2024-11-21 03:39:50,782 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-21 03:39:50,806 main DEBUG Took 0.019325 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-21 03:39:50,806 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-21 03:39:50,806 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-21 03:39:50,808 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-21 03:39:50,810 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:39:50,822 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-21 03:39:50,841 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:39:50,843 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:39:50,844 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:39:50,844 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:39:50,845 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:39:50,846 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:39:50,847 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:39:50,848 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:39:50,849 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:39:50,849 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:39:50,851 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:39:50,851 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:39:50,852 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:39:50,853 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:39:50,853 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:39:50,854 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:39:50,855 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:39:50,855 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:39:50,856 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:39:50,856 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:39:50,857 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:39:50,858 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:39:50,858 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:39:50,859 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 03:39:50,860 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:39:50,860 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-21 03:39:50,862 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 03:39:50,864 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-21 03:39:50,867 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-21 03:39:50,868 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-21 03:39:50,870 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-21 03:39:50,871 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-21 03:39:50,881 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-21 03:39:50,884 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-21 03:39:50,886 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-21 03:39:50,887 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-21 03:39:50,888 main DEBUG createAppenders(={Console}) 2024-11-21 03:39:50,888 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-21 03:39:50,889 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-21 03:39:50,889 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-21 03:39:50,890 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-21 03:39:50,890 main DEBUG OutputStream closed 2024-11-21 03:39:50,891 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-21 03:39:50,891 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-21 03:39:50,891 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-21 03:39:51,003 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-21 03:39:51,007 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-21 03:39:51,008 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-21 03:39:51,010 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-21 03:39:51,011 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-21 03:39:51,012 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-21 03:39:51,012 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-21 03:39:51,012 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-21 03:39:51,013 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-21 03:39:51,013 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-21 03:39:51,015 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-21 03:39:51,015 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-21 03:39:51,016 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-21 03:39:51,016 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-21 03:39:51,016 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-21 03:39:51,017 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-21 03:39:51,017 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-21 03:39:51,018 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-21 03:39:51,021 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21 03:39:51,021 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-21 03:39:51,022 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-21 03:39:51,023 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-21T03:39:51,387 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537 2024-11-21 03:39:51,390 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-21 03:39:51,391 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21T03:39:51,401 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-21T03:39:51,428 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-21T03:39:51,451 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/cluster_9cc19675-7463-4642-6f00-3e0c0079ede6, deleteOnExit=true 2024-11-21T03:39:51,452 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-21T03:39:51,453 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/test.cache.data in system properties and HBase conf 2024-11-21T03:39:51,454 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/hadoop.tmp.dir in system properties and HBase conf 2024-11-21T03:39:51,455 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/hadoop.log.dir in system properties and HBase conf 2024-11-21T03:39:51,457 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-21T03:39:51,458 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-21T03:39:51,458 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-21T03:39:51,594 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-21T03:39:51,739 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T03:39:51,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-21T03:39:51,746 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-21T03:39:51,746 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-21T03:39:51,747 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T03:39:51,748 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-21T03:39:51,749 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-21T03:39:51,754 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T03:39:51,755 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T03:39:51,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-21T03:39:51,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/nfs.dump.dir in system properties and HBase conf 2024-11-21T03:39:51,761 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/java.io.tmpdir in system properties and HBase conf 2024-11-21T03:39:51,761 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T03:39:51,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-21T03:39:51,771 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-21T03:39:52,978 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-21T03:39:53,071 INFO [Time-limited test {}] log.Log(170): Logging initialized @3168ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-21T03:39:53,159 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T03:39:53,229 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T03:39:53,271 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T03:39:53,271 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T03:39:53,273 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-21T03:39:53,290 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T03:39:53,293 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/hadoop.log.dir/,AVAILABLE} 2024-11-21T03:39:53,295 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T03:39:53,536 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/java.io.tmpdir/jetty-localhost-34377-hadoop-hdfs-3_4_1-tests_jar-_-any-3028549612741805713/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-21T03:39:53,542 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:34377} 2024-11-21T03:39:53,542 INFO [Time-limited test {}] server.Server(415): Started @3641ms 2024-11-21T03:39:54,134 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T03:39:54,142 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T03:39:54,144 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T03:39:54,144 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T03:39:54,144 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-21T03:39:54,145 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/hadoop.log.dir/,AVAILABLE} 2024-11-21T03:39:54,146 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@47c7fd6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T03:39:54,277 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@250cb724{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/java.io.tmpdir/jetty-localhost-46599-hadoop-hdfs-3_4_1-tests_jar-_-any-8580946905452960089/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T03:39:54,278 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@69edd336{HTTP/1.1, (http/1.1)}{localhost:46599} 2024-11-21T03:39:54,278 INFO [Time-limited test {}] server.Server(415): Started @4376ms 2024-11-21T03:39:54,325 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-21T03:39:55,118 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/cluster_9cc19675-7463-4642-6f00-3e0c0079ede6/data/data2/current/BP-40659483-172.17.0.2-1732160392521/current, will proceed with Du for space computation calculation, 2024-11-21T03:39:55,118 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/cluster_9cc19675-7463-4642-6f00-3e0c0079ede6/data/data1/current/BP-40659483-172.17.0.2-1732160392521/current, will proceed with Du for space computation calculation, 2024-11-21T03:39:55,147 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-21T03:39:55,191 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf9a4f23c517fe030 with lease ID 0x6470cc40e0656dd1: Processing first storage report for DS-2d2aa963-0848-48a9-af98-b7b4ee64f7c6 from datanode DatanodeRegistration(127.0.0.1:38147, datanodeUuid=0b26e3ee-0cc4-406b-b7a6-96e43e6b6dd8, infoPort=33005, infoSecurePort=0, ipcPort=39513, storageInfo=lv=-57;cid=testClusterID;nsid=702141155;c=1732160392521) 2024-11-21T03:39:55,193 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf9a4f23c517fe030 with lease ID 0x6470cc40e0656dd1: from storage DS-2d2aa963-0848-48a9-af98-b7b4ee64f7c6 node DatanodeRegistration(127.0.0.1:38147, datanodeUuid=0b26e3ee-0cc4-406b-b7a6-96e43e6b6dd8, infoPort=33005, infoSecurePort=0, ipcPort=39513, storageInfo=lv=-57;cid=testClusterID;nsid=702141155;c=1732160392521), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-21T03:39:55,193 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf9a4f23c517fe030 with lease ID 0x6470cc40e0656dd1: Processing first storage report for DS-698ee7cd-5bcf-4ead-8bd7-5c9dec025b0c from datanode DatanodeRegistration(127.0.0.1:38147, datanodeUuid=0b26e3ee-0cc4-406b-b7a6-96e43e6b6dd8, infoPort=33005, infoSecurePort=0, ipcPort=39513, storageInfo=lv=-57;cid=testClusterID;nsid=702141155;c=1732160392521) 2024-11-21T03:39:55,194 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf9a4f23c517fe030 with lease ID 0x6470cc40e0656dd1: from storage DS-698ee7cd-5bcf-4ead-8bd7-5c9dec025b0c node DatanodeRegistration(127.0.0.1:38147, datanodeUuid=0b26e3ee-0cc4-406b-b7a6-96e43e6b6dd8, infoPort=33005, infoSecurePort=0, ipcPort=39513, storageInfo=lv=-57;cid=testClusterID;nsid=702141155;c=1732160392521), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T03:39:55,194 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537 2024-11-21T03:39:55,265 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/cluster_9cc19675-7463-4642-6f00-3e0c0079ede6/zookeeper_0, clientPort=53280, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/cluster_9cc19675-7463-4642-6f00-3e0c0079ede6/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/cluster_9cc19675-7463-4642-6f00-3e0c0079ede6/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-21T03:39:55,276 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53280 2024-11-21T03:39:55,289 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T03:39:55,292 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T03:39:55,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741825_1001 (size=7) 2024-11-21T03:39:55,904 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0 with version=8 2024-11-21T03:39:55,905 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/hbase-staging 2024-11-21T03:39:55,985 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-21T03:39:56,226 INFO [Time-limited test {}] client.ConnectionUtils(128): master/fdc0a393186c:0 server-side Connection retries=45 2024-11-21T03:39:56,234 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T03:39:56,235 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T03:39:56,239 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T03:39:56,239 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T03:39:56,239 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T03:39:56,395 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-21T03:39:56,472 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-21T03:39:56,484 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-21T03:39:56,489 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T03:39:56,520 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1540 (auto-detected) 2024-11-21T03:39:56,521 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-21T03:39:56,546 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44285 2024-11-21T03:39:56,576 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44285 connecting to ZooKeeper ensemble=127.0.0.1:53280 2024-11-21T03:39:56,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:442850x0, quorum=127.0.0.1:53280, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T03:39:56,675 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44285-0x1015cc3aab20000 connected 2024-11-21T03:39:56,769 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T03:39:56,773 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T03:39:56,785 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44285-0x1015cc3aab20000, quorum=127.0.0.1:53280, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T03:39:56,790 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0, hbase.cluster.distributed=false 2024-11-21T03:39:56,814 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44285-0x1015cc3aab20000, quorum=127.0.0.1:53280, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T03:39:56,818 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44285 2024-11-21T03:39:56,821 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44285 2024-11-21T03:39:56,821 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44285 2024-11-21T03:39:56,822 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44285 2024-11-21T03:39:56,822 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44285 2024-11-21T03:39:56,926 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fdc0a393186c:0 server-side Connection retries=45 2024-11-21T03:39:56,928 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T03:39:56,929 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T03:39:56,929 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T03:39:56,929 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T03:39:56,930 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T03:39:56,933 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T03:39:56,936 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T03:39:56,937 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34631 2024-11-21T03:39:56,941 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34631 connecting to ZooKeeper ensemble=127.0.0.1:53280 2024-11-21T03:39:56,943 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T03:39:56,948 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T03:39:56,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:346310x0, quorum=127.0.0.1:53280, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T03:39:56,973 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34631-0x1015cc3aab20001 connected 2024-11-21T03:39:56,973 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34631-0x1015cc3aab20001, quorum=127.0.0.1:53280, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T03:39:56,977 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T03:39:56,989 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T03:39:56,992 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34631-0x1015cc3aab20001, quorum=127.0.0.1:53280, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T03:39:56,998 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34631-0x1015cc3aab20001, quorum=127.0.0.1:53280, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T03:39:57,003 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34631 2024-11-21T03:39:57,004 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34631 2024-11-21T03:39:57,006 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34631 2024-11-21T03:39:57,010 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34631 2024-11-21T03:39:57,010 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34631 2024-11-21T03:39:57,027 DEBUG [M:0;fdc0a393186c:44285 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fdc0a393186c:44285 2024-11-21T03:39:57,028 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fdc0a393186c,44285,1732160396080 2024-11-21T03:39:57,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44285-0x1015cc3aab20000, quorum=127.0.0.1:53280, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T03:39:57,063 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34631-0x1015cc3aab20001, quorum=127.0.0.1:53280, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T03:39:57,065 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44285-0x1015cc3aab20000, quorum=127.0.0.1:53280, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fdc0a393186c,44285,1732160396080 2024-11-21T03:39:57,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34631-0x1015cc3aab20001, quorum=127.0.0.1:53280, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T03:39:57,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34631-0x1015cc3aab20001, quorum=127.0.0.1:53280, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:39:57,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44285-0x1015cc3aab20000, quorum=127.0.0.1:53280, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:39:57,094 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44285-0x1015cc3aab20000, quorum=127.0.0.1:53280, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T03:39:57,095 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fdc0a393186c,44285,1732160396080 from backup master directory 2024-11-21T03:39:57,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44285-0x1015cc3aab20000, quorum=127.0.0.1:53280, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fdc0a393186c,44285,1732160396080 2024-11-21T03:39:57,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34631-0x1015cc3aab20001, quorum=127.0.0.1:53280, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T03:39:57,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44285-0x1015cc3aab20000, quorum=127.0.0.1:53280, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T03:39:57,105 WARN [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T03:39:57,105 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fdc0a393186c,44285,1732160396080 2024-11-21T03:39:57,108 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-21T03:39:57,110 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-21T03:39:57,173 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/hbase.id] with ID: 1afa9799-9c29-4101-98df-fbf56a76f01a 2024-11-21T03:39:57,173 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/.tmp/hbase.id 2024-11-21T03:39:57,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741826_1002 (size=42) 2024-11-21T03:39:57,592 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/.tmp/hbase.id]:[hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/hbase.id] 2024-11-21T03:39:57,641 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T03:39:57,646 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-21T03:39:57,663 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-11-21T03:39:57,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34631-0x1015cc3aab20001, quorum=127.0.0.1:53280, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:39:57,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44285-0x1015cc3aab20000, quorum=127.0.0.1:53280, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:39:57,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741827_1003 (size=196) 2024-11-21T03:39:58,149 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T03:39:58,152 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-21T03:39:58,165 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.<clinit>(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.<init>(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.<init>(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:39:58,169 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T03:39:58,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741828_1004 (size=1189) 2024-11-21T03:39:58,624 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/MasterData/data/master/store 2024-11-21T03:39:58,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741829_1005 (size=34) 2024-11-21T03:39:59,055 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-21T03:39:59,058 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:39:59,059 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T03:39:59,059 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:39:59,059 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:39:59,060 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T03:39:59,060 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:39:59,061 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T03:39:59,061 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732160399059Disabling compacts and flushes for region at 1732160399059Disabling writes for close at 1732160399060 (+1 ms)Writing region close event to WAL at 1732160399060Closed at 1732160399060 2024-11-21T03:39:59,063 WARN [master/fdc0a393186c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/MasterData/data/master/store/.initializing 2024-11-21T03:39:59,064 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/MasterData/WALs/fdc0a393186c,44285,1732160396080 2024-11-21T03:39:59,071 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T03:39:59,084 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fdc0a393186c%2C44285%2C1732160396080, suffix=, logDir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/MasterData/WALs/fdc0a393186c,44285,1732160396080, archiveDir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/MasterData/oldWALs, maxLogs=10 2024-11-21T03:39:59,112 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/MasterData/WALs/fdc0a393186c,44285,1732160396080/fdc0a393186c%2C44285%2C1732160396080.1732160399089, exclude list is [], retry=0 2024-11-21T03:39:59,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-2d2aa963-0848-48a9-af98-b7b4ee64f7c6,DISK] 2024-11-21T03:39:59,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-21T03:39:59,160 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/MasterData/WALs/fdc0a393186c,44285,1732160396080/fdc0a393186c%2C44285%2C1732160396080.1732160399089 2024-11-21T03:39:59,161 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33005:33005)] 2024-11-21T03:39:59,161 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:39:59,162 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:39:59,164 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:39:59,165 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:39:59,200 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:39:59,221 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-21T03:39:59,224 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:39:59,227 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:39:59,227 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:39:59,230 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-21T03:39:59,230 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:39:59,231 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:39:59,232 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:39:59,234 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-21T03:39:59,235 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:39:59,235 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:39:59,236 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:39:59,238 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-21T03:39:59,238 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:39:59,239 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:39:59,240 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:39:59,244 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:39:59,245 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:39:59,251 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:39:59,252 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:39:59,255 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T03:39:59,259 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T03:39:59,263 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T03:39:59,265 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63322826, jitterRate=-0.05641636252403259}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T03:39:59,274 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732160399176Initializing all the Stores at 1732160399178 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732160399179 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732160399179Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732160399180 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732160399180Cleaning up temporary data from old regions at 1732160399252 (+72 ms)Region opened successfully at 1732160399273 (+21 ms) 2024-11-21T03:39:59,275 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-21T03:39:59,313 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4be0e44b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fdc0a393186c/172.17.0.2:0 2024-11-21T03:39:59,354 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-21T03:39:59,367 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-21T03:39:59,367 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-21T03:39:59,370 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T03:39:59,372 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-21T03:39:59,378 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-21T03:39:59,378 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-21T03:39:59,410 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-21T03:39:59,423 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44285-0x1015cc3aab20000, quorum=127.0.0.1:53280, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-21T03:39:59,475 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-21T03:39:59,479 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-21T03:39:59,481 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44285-0x1015cc3aab20000, quorum=127.0.0.1:53280, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-21T03:39:59,491 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-21T03:39:59,495 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-21T03:39:59,499 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44285-0x1015cc3aab20000, quorum=127.0.0.1:53280, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-21T03:39:59,508 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-21T03:39:59,510 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44285-0x1015cc3aab20000, quorum=127.0.0.1:53280, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-21T03:39:59,520 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-21T03:39:59,538 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44285-0x1015cc3aab20000, quorum=127.0.0.1:53280, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-21T03:39:59,545 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-21T03:39:59,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34631-0x1015cc3aab20001, quorum=127.0.0.1:53280, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T03:39:59,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44285-0x1015cc3aab20000, quorum=127.0.0.1:53280, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T03:39:59,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34631-0x1015cc3aab20001, quorum=127.0.0.1:53280, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:39:59,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44285-0x1015cc3aab20000, quorum=127.0.0.1:53280, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:39:59,562 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fdc0a393186c,44285,1732160396080, sessionid=0x1015cc3aab20000, setting cluster-up flag (Was=false) 2024-11-21T03:39:59,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34631-0x1015cc3aab20001, quorum=127.0.0.1:53280, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:39:59,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44285-0x1015cc3aab20000, quorum=127.0.0.1:53280, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:39:59,617 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-21T03:39:59,619 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fdc0a393186c,44285,1732160396080 2024-11-21T03:39:59,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34631-0x1015cc3aab20001, quorum=127.0.0.1:53280, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:39:59,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44285-0x1015cc3aab20000, quorum=127.0.0.1:53280, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:39:59,671 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-21T03:39:59,673 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fdc0a393186c,44285,1732160396080 2024-11-21T03:39:59,680 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-21T03:39:59,719 INFO [RS:0;fdc0a393186c:34631 {}] regionserver.HRegionServer(746): ClusterId : 1afa9799-9c29-4101-98df-fbf56a76f01a 2024-11-21T03:39:59,722 DEBUG [RS:0;fdc0a393186c:34631 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T03:39:59,756 DEBUG [RS:0;fdc0a393186c:34631 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T03:39:59,756 DEBUG [RS:0;fdc0a393186c:34631 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T03:39:59,768 DEBUG [RS:0;fdc0a393186c:34631 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T03:39:59,769 DEBUG [RS:0;fdc0a393186c:34631 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38642c55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fdc0a393186c/172.17.0.2:0 2024-11-21T03:39:59,773 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T03:39:59,784 DEBUG [RS:0;fdc0a393186c:34631 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fdc0a393186c:34631 2024-11-21T03:39:59,785 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-21T03:39:59,787 INFO [RS:0;fdc0a393186c:34631 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T03:39:59,788 INFO [RS:0;fdc0a393186c:34631 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T03:39:59,788 DEBUG [RS:0;fdc0a393186c:34631 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T03:39:59,791 INFO [RS:0;fdc0a393186c:34631 {}] regionserver.HRegionServer(2659): reportForDuty to master=fdc0a393186c,44285,1732160396080 with port=34631, startcode=1732160396889 2024-11-21T03:39:59,793 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-21T03:39:59,801 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fdc0a393186c,44285,1732160396080 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-21T03:39:59,807 DEBUG [RS:0;fdc0a393186c:34631 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T03:39:59,811 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fdc0a393186c:0, corePoolSize=5, maxPoolSize=5 2024-11-21T03:39:59,811 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fdc0a393186c:0, corePoolSize=5, maxPoolSize=5 2024-11-21T03:39:59,811 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fdc0a393186c:0, corePoolSize=5, maxPoolSize=5 2024-11-21T03:39:59,812 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fdc0a393186c:0, corePoolSize=5, maxPoolSize=5 2024-11-21T03:39:59,812 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fdc0a393186c:0, corePoolSize=10, maxPoolSize=10 2024-11-21T03:39:59,812 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:39:59,812 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fdc0a393186c:0, corePoolSize=2, maxPoolSize=2 2024-11-21T03:39:59,812 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:39:59,822 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T03:39:59,823 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-21T03:39:59,825 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732160429825 2024-11-21T03:39:59,827 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-21T03:39:59,829 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-21T03:39:59,830 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:39:59,830 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-21T03:39:59,835 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-21T03:39:59,835 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-21T03:39:59,836 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-21T03:39:59,836 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-21T03:39:59,850 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T03:39:59,859 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-21T03:39:59,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741831_1007 (size=1321) 2024-11-21T03:39:59,861 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-21T03:39:59,861 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-21T03:39:59,862 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49923, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T03:39:59,862 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-21T03:39:59,863 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0 2024-11-21T03:39:59,867 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-21T03:39:59,868 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-21T03:39:59,871 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44285 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:39:59,873 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fdc0a393186c:0:becomeActiveMaster-HFileCleaner.large.0-1732160399870,5,FailOnTimeoutGroup] 2024-11-21T03:39:59,879 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fdc0a393186c:0:becomeActiveMaster-HFileCleaner.small.0-1732160399874,5,FailOnTimeoutGroup] 2024-11-21T03:39:59,879 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T03:39:59,880 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-21T03:39:59,881 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-21T03:39:59,882 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-21T03:39:59,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741832_1008 (size=32) 2024-11-21T03:39:59,905 DEBUG [RS:0;fdc0a393186c:34631 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-21T03:39:59,905 WARN [RS:0;fdc0a393186c:34631 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-21T03:40:00,007 INFO [RS:0;fdc0a393186c:34631 {}] regionserver.HRegionServer(2659): reportForDuty to master=fdc0a393186c,44285,1732160396080 with port=34631, startcode=1732160396889 2024-11-21T03:40:00,009 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44285 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fdc0a393186c,34631,1732160396889 2024-11-21T03:40:00,011 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44285 {}] master.ServerManager(517): Registering regionserver=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:00,019 DEBUG [RS:0;fdc0a393186c:34631 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0 2024-11-21T03:40:00,019 DEBUG [RS:0;fdc0a393186c:34631 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45067 2024-11-21T03:40:00,019 DEBUG [RS:0;fdc0a393186c:34631 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T03:40:00,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44285-0x1015cc3aab20000, quorum=127.0.0.1:53280, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T03:40:00,050 DEBUG [RS:0;fdc0a393186c:34631 {}] zookeeper.ZKUtil(111): regionserver:34631-0x1015cc3aab20001, quorum=127.0.0.1:53280, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fdc0a393186c,34631,1732160396889 2024-11-21T03:40:00,050 WARN [RS:0;fdc0a393186c:34631 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T03:40:00,051 INFO [RS:0;fdc0a393186c:34631 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T03:40:00,051 DEBUG [RS:0;fdc0a393186c:34631 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/WALs/fdc0a393186c,34631,1732160396889 2024-11-21T03:40:00,052 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fdc0a393186c,34631,1732160396889] 2024-11-21T03:40:00,075 INFO [RS:0;fdc0a393186c:34631 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T03:40:00,090 INFO [RS:0;fdc0a393186c:34631 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T03:40:00,094 INFO [RS:0;fdc0a393186c:34631 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T03:40:00,094 INFO [RS:0;fdc0a393186c:34631 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T03:40:00,095 INFO [RS:0;fdc0a393186c:34631 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T03:40:00,100 INFO [RS:0;fdc0a393186c:34631 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T03:40:00,101 INFO [RS:0;fdc0a393186c:34631 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T03:40:00,101 DEBUG [RS:0;fdc0a393186c:34631 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:40:00,101 DEBUG [RS:0;fdc0a393186c:34631 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:40:00,101 DEBUG [RS:0;fdc0a393186c:34631 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:40:00,102 DEBUG [RS:0;fdc0a393186c:34631 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:40:00,102 DEBUG [RS:0;fdc0a393186c:34631 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:40:00,102 DEBUG [RS:0;fdc0a393186c:34631 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fdc0a393186c:0, corePoolSize=2, maxPoolSize=2 2024-11-21T03:40:00,102 DEBUG [RS:0;fdc0a393186c:34631 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:40:00,102 DEBUG [RS:0;fdc0a393186c:34631 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:40:00,102 DEBUG [RS:0;fdc0a393186c:34631 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:40:00,102 DEBUG [RS:0;fdc0a393186c:34631 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:40:00,102 DEBUG [RS:0;fdc0a393186c:34631 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:40:00,103 DEBUG [RS:0;fdc0a393186c:34631 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fdc0a393186c:0, corePoolSize=1, maxPoolSize=1 2024-11-21T03:40:00,103 DEBUG [RS:0;fdc0a393186c:34631 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fdc0a393186c:0, corePoolSize=3, maxPoolSize=3 2024-11-21T03:40:00,103 DEBUG [RS:0;fdc0a393186c:34631 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0, corePoolSize=3, maxPoolSize=3 2024-11-21T03:40:00,104 INFO [RS:0;fdc0a393186c:34631 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T03:40:00,104 INFO [RS:0;fdc0a393186c:34631 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T03:40:00,104 INFO [RS:0;fdc0a393186c:34631 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T03:40:00,104 INFO [RS:0;fdc0a393186c:34631 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T03:40:00,104 INFO [RS:0;fdc0a393186c:34631 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T03:40:00,105 INFO [RS:0;fdc0a393186c:34631 {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,34631,1732160396889-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T03:40:00,123 INFO [RS:0;fdc0a393186c:34631 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T03:40:00,125 INFO [RS:0;fdc0a393186c:34631 {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,34631,1732160396889-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T03:40:00,125 INFO [RS:0;fdc0a393186c:34631 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:40:00,125 INFO [RS:0;fdc0a393186c:34631 {}] regionserver.Replication(171): fdc0a393186c,34631,1732160396889 started 2024-11-21T03:40:00,141 INFO [RS:0;fdc0a393186c:34631 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:40:00,142 INFO [RS:0;fdc0a393186c:34631 {}] regionserver.HRegionServer(1482): Serving as fdc0a393186c,34631,1732160396889, RpcServer on fdc0a393186c/172.17.0.2:34631, sessionid=0x1015cc3aab20001 2024-11-21T03:40:00,142 DEBUG [RS:0;fdc0a393186c:34631 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T03:40:00,142 DEBUG [RS:0;fdc0a393186c:34631 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fdc0a393186c,34631,1732160396889 2024-11-21T03:40:00,143 DEBUG [RS:0;fdc0a393186c:34631 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fdc0a393186c,34631,1732160396889' 2024-11-21T03:40:00,143 DEBUG [RS:0;fdc0a393186c:34631 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T03:40:00,144 DEBUG [RS:0;fdc0a393186c:34631 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T03:40:00,144 DEBUG [RS:0;fdc0a393186c:34631 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T03:40:00,144 DEBUG [RS:0;fdc0a393186c:34631 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T03:40:00,145 DEBUG [RS:0;fdc0a393186c:34631 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fdc0a393186c,34631,1732160396889 2024-11-21T03:40:00,145 DEBUG [RS:0;fdc0a393186c:34631 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fdc0a393186c,34631,1732160396889' 2024-11-21T03:40:00,145 DEBUG [RS:0;fdc0a393186c:34631 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T03:40:00,145 DEBUG [RS:0;fdc0a393186c:34631 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T03:40:00,146 DEBUG [RS:0;fdc0a393186c:34631 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T03:40:00,146 INFO [RS:0;fdc0a393186c:34631 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T03:40:00,146 INFO [RS:0;fdc0a393186c:34631 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T03:40:00,255 INFO [RS:0;fdc0a393186c:34631 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T03:40:00,261 INFO [RS:0;fdc0a393186c:34631 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fdc0a393186c%2C34631%2C1732160396889, suffix=, logDir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/WALs/fdc0a393186c,34631,1732160396889, archiveDir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/oldWALs, maxLogs=32 2024-11-21T03:40:00,275 DEBUG [RS:0;fdc0a393186c:34631 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/WALs/fdc0a393186c,34631,1732160396889/fdc0a393186c%2C34631%2C1732160396889.1732160400263, exclude list is [], retry=0 2024-11-21T03:40:00,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-2d2aa963-0848-48a9-af98-b7b4ee64f7c6,DISK] 2024-11-21T03:40:00,283 INFO [RS:0;fdc0a393186c:34631 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/WALs/fdc0a393186c,34631,1732160396889/fdc0a393186c%2C34631%2C1732160396889.1732160400263 2024-11-21T03:40:00,283 DEBUG [RS:0;fdc0a393186c:34631 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33005:33005)] 2024-11-21T03:40:00,299 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:40:00,302 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T03:40:00,304 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T03:40:00,304 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:00,305 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:40:00,305 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T03:40:00,308 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T03:40:00,308 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:00,309 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:40:00,310 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T03:40:00,313 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T03:40:00,313 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:00,314 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:40:00,314 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T03:40:00,317 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T03:40:00,317 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:00,318 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:40:00,319 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T03:40:00,320 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/hbase/meta/1588230740 2024-11-21T03:40:00,321 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/hbase/meta/1588230740 2024-11-21T03:40:00,324 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T03:40:00,324 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T03:40:00,325 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T03:40:00,327 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T03:40:00,331 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T03:40:00,332 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59215060, jitterRate=-0.1176268458366394}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T03:40:00,335 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732160400299Initializing all the Stores at 1732160400301 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732160400301Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732160400301Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732160400301Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732160400301Cleaning up temporary data from old regions at 1732160400324 (+23 ms)Region opened successfully at 1732160400335 (+11 ms) 2024-11-21T03:40:00,335 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T03:40:00,335 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T03:40:00,335 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T03:40:00,335 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T03:40:00,335 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T03:40:00,337 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T03:40:00,337 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732160400335Disabling compacts and flushes for region at 1732160400335Disabling writes for close at 1732160400335Writing region close event to WAL at 1732160400336 (+1 ms)Closed at 1732160400336 2024-11-21T03:40:00,340 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T03:40:00,340 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-21T03:40:00,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T03:40:00,353 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T03:40:00,356 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-21T03:40:00,510 DEBUG [fdc0a393186c:44285 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-21T03:40:00,523 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:00,529 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fdc0a393186c,34631,1732160396889, state=OPENING 2024-11-21T03:40:00,583 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-21T03:40:00,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44285-0x1015cc3aab20000, quorum=127.0.0.1:53280, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:40:00,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34631-0x1015cc3aab20001, quorum=127.0.0.1:53280, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T03:40:00,598 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T03:40:00,598 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T03:40:00,600 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T03:40:00,604 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fdc0a393186c,34631,1732160396889}] 2024-11-21T03:40:00,784 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T03:40:00,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51413, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T03:40:00,797 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T03:40:00,797 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T03:40:00,798 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T03:40:00,801 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fdc0a393186c%2C34631%2C1732160396889.meta, suffix=.meta, logDir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/WALs/fdc0a393186c,34631,1732160396889, archiveDir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/oldWALs, maxLogs=32 2024-11-21T03:40:00,817 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/WALs/fdc0a393186c,34631,1732160396889/fdc0a393186c%2C34631%2C1732160396889.meta.1732160400803.meta, exclude list is [], retry=0 2024-11-21T03:40:00,821 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38147,DS-2d2aa963-0848-48a9-af98-b7b4ee64f7c6,DISK] 2024-11-21T03:40:00,824 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/WALs/fdc0a393186c,34631,1732160396889/fdc0a393186c%2C34631%2C1732160396889.meta.1732160400803.meta 2024-11-21T03:40:00,825 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33005:33005)] 2024-11-21T03:40:00,825 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:40:00,827 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T03:40:00,829 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T03:40:00,833 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T03:40:00,836 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T03:40:00,837 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:40:00,837 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T03:40:00,837 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T03:40:00,840 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T03:40:00,842 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T03:40:00,842 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:00,843 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:40:00,843 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T03:40:00,844 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T03:40:00,844 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:00,845 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:40:00,846 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T03:40:00,847 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T03:40:00,847 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:00,848 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:40:00,848 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T03:40:00,849 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T03:40:00,850 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:00,850 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T03:40:00,851 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T03:40:00,852 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/hbase/meta/1588230740 2024-11-21T03:40:00,855 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/hbase/meta/1588230740 2024-11-21T03:40:00,858 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T03:40:00,858 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T03:40:00,859 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T03:40:00,861 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T03:40:00,863 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70825789, jitterRate=0.0553864985704422}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T03:40:00,863 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T03:40:00,865 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732160400838Writing region info on filesystem at 1732160400838Initializing all the Stores at 1732160400840 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732160400840Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732160400840Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732160400840Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732160400840Cleaning up temporary data from old regions at 1732160400858 (+18 ms)Running coprocessor post-open hooks at 1732160400863 (+5 ms)Region opened successfully at 1732160400864 (+1 ms) 2024-11-21T03:40:00,871 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732160400775 2024-11-21T03:40:00,881 DEBUG [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T03:40:00,882 INFO [RS_OPEN_META-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T03:40:00,883 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:00,885 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fdc0a393186c,34631,1732160396889, state=OPEN 2024-11-21T03:40:00,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34631-0x1015cc3aab20001, quorum=127.0.0.1:53280, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T03:40:00,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44285-0x1015cc3aab20000, quorum=127.0.0.1:53280, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T03:40:00,936 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T03:40:00,936 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T03:40:00,937 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:00,942 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-21T03:40:00,942 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fdc0a393186c,34631,1732160396889 in 334 msec 2024-11-21T03:40:00,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-21T03:40:00,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 599 msec 2024-11-21T03:40:00,950 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T03:40:00,950 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-21T03:40:00,970 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:00,971 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:00,987 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:00,990 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58701, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:01,023 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3010 sec 2024-11-21T03:40:01,024 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732160401024, completionTime=-1 2024-11-21T03:40:01,038 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-21T03:40:01,038 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T03:40:01,065 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-21T03:40:01,065 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732160461065 2024-11-21T03:40:01,066 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732160521065 2024-11-21T03:40:01,066 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-21T03:40:01,068 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,44285,1732160396080-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T03:40:01,068 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,44285,1732160396080-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:40:01,068 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,44285,1732160396080-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:40:01,069 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fdc0a393186c:44285, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:40:01,070 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-21T03:40:01,070 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-21T03:40:01,078 DEBUG [master/fdc0a393186c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-21T03:40:01,093 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.987sec 2024-11-21T03:40:01,094 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-21T03:40:01,095 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-21T03:40:01,096 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-21T03:40:01,097 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-21T03:40:01,097 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-21T03:40:01,098 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,44285,1732160396080-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T03:40:01,098 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,44285,1732160396080-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-21T03:40:01,107 DEBUG [master/fdc0a393186c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-21T03:40:01,108 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-21T03:40:01,108 INFO [master/fdc0a393186c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fdc0a393186c,44285,1732160396080-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T03:40:01,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@609db4ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:01,131 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-21T03:40:01,131 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-21T03:40:01,133 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:01,135 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:01,142 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:01,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:01,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:01,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5db9d8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:01,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:01,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:01,150 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:01,153 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366e166a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:01,153 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:01,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:01,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:01,162 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:01,182 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fdc0a393186c,44285,1732160396080 2024-11-21T03:40:01,197 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=388, ProcessCount=11, AvailableMemoryMB=7264 2024-11-21T03:40:01,204 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T03:40:01,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is fdc0a393186c,44285,1732160396080 2024-11-21T03:40:01,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5b65baa9 2024-11-21T03:40:01,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T03:40:01,214 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T03:40:01,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T03:40:01,220 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T03:40:01,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T03:40:01,229 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T03:40:01,231 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-21T03:40:01,231 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:01,233 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T03:40:01,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T03:40:01,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741835_1011 (size=959) 2024-11-21T03:40:01,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T03:40:01,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T03:40:01,652 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0 2024-11-21T03:40:01,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741836_1012 (size=53) 2024-11-21T03:40:01,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T03:40:02,067 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:40:02,067 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 30740eafbeb6526d1c29b0b1fec56da6, disabling compactions & flushes 2024-11-21T03:40:02,068 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:02,068 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:02,068 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. after waiting 0 ms 2024-11-21T03:40:02,068 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:02,068 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:02,068 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 30740eafbeb6526d1c29b0b1fec56da6: Waiting for close lock at 1732160402067Disabling compacts and flushes for region at 1732160402067Disabling writes for close at 1732160402068 (+1 ms)Writing region close event to WAL at 1732160402068Closed at 1732160402068 2024-11-21T03:40:02,071 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T03:40:02,075 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732160402071"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732160402071"}]},"ts":"1732160402071"} 2024-11-21T03:40:02,080 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T03:40:02,083 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T03:40:02,086 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732160402083"}]},"ts":"1732160402083"} 2024-11-21T03:40:02,092 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T03:40:02,094 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=30740eafbeb6526d1c29b0b1fec56da6, ASSIGN}] 2024-11-21T03:40:02,097 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=30740eafbeb6526d1c29b0b1fec56da6, ASSIGN 2024-11-21T03:40:02,100 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=30740eafbeb6526d1c29b0b1fec56da6, ASSIGN; state=OFFLINE, location=fdc0a393186c,34631,1732160396889; forceNewPlan=false, retain=false 2024-11-21T03:40:02,252 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=30740eafbeb6526d1c29b0b1fec56da6, regionState=OPENING, regionLocation=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:02,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=30740eafbeb6526d1c29b0b1fec56da6, ASSIGN because future has completed 2024-11-21T03:40:02,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889}] 2024-11-21T03:40:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T03:40:02,420 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:02,421 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:40:02,421 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:02,421 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:40:02,421 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:02,421 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:02,423 INFO [StoreOpener-30740eafbeb6526d1c29b0b1fec56da6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:02,426 INFO [StoreOpener-30740eafbeb6526d1c29b0b1fec56da6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30740eafbeb6526d1c29b0b1fec56da6 columnFamilyName A 2024-11-21T03:40:02,426 DEBUG [StoreOpener-30740eafbeb6526d1c29b0b1fec56da6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:02,427 INFO [StoreOpener-30740eafbeb6526d1c29b0b1fec56da6-1 {}] regionserver.HStore(327): Store=30740eafbeb6526d1c29b0b1fec56da6/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:40:02,427 INFO [StoreOpener-30740eafbeb6526d1c29b0b1fec56da6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:02,429 INFO [StoreOpener-30740eafbeb6526d1c29b0b1fec56da6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30740eafbeb6526d1c29b0b1fec56da6 columnFamilyName B 2024-11-21T03:40:02,429 DEBUG [StoreOpener-30740eafbeb6526d1c29b0b1fec56da6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:02,430 INFO [StoreOpener-30740eafbeb6526d1c29b0b1fec56da6-1 {}] regionserver.HStore(327): Store=30740eafbeb6526d1c29b0b1fec56da6/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:40:02,430 INFO [StoreOpener-30740eafbeb6526d1c29b0b1fec56da6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:02,434 INFO [StoreOpener-30740eafbeb6526d1c29b0b1fec56da6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30740eafbeb6526d1c29b0b1fec56da6 columnFamilyName C 2024-11-21T03:40:02,434 DEBUG [StoreOpener-30740eafbeb6526d1c29b0b1fec56da6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:02,435 INFO [StoreOpener-30740eafbeb6526d1c29b0b1fec56da6-1 {}] regionserver.HStore(327): Store=30740eafbeb6526d1c29b0b1fec56da6/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:40:02,436 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:02,438 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:02,439 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:02,442 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:02,442 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:02,444 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T03:40:02,447 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:02,450 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T03:40:02,451 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 30740eafbeb6526d1c29b0b1fec56da6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71863644, jitterRate=0.07085174322128296}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T03:40:02,451 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:02,452 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 30740eafbeb6526d1c29b0b1fec56da6: Running coprocessor pre-open hook at 1732160402421Writing region info on filesystem at 1732160402421Initializing all the Stores at 1732160402423 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732160402423Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732160402423Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732160402423Cleaning up temporary data from old regions at 1732160402442 (+19 ms)Running coprocessor post-open hooks at 1732160402451 (+9 ms)Region opened successfully at 1732160402452 (+1 ms) 2024-11-21T03:40:02,455 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., pid=6, masterSystemTime=1732160402414 2024-11-21T03:40:02,460 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:02,460 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:02,462 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=30740eafbeb6526d1c29b0b1fec56da6, regionState=OPEN, openSeqNum=2, regionLocation=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:02,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 because future has completed 2024-11-21T03:40:02,474 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-21T03:40:02,475 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 in 211 msec 2024-11-21T03:40:02,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-21T03:40:02,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=30740eafbeb6526d1c29b0b1fec56da6, ASSIGN in 381 msec 2024-11-21T03:40:02,483 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T03:40:02,483 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732160402483"}]},"ts":"1732160402483"} 2024-11-21T03:40:02,488 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T03:40:02,490 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T03:40:02,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2680 sec 2024-11-21T03:40:03,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T03:40:03,391 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:03,401 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@710944e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,401 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:03,401 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:03,403 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:03,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:03,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:03,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33c6058d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:03,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:03,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:03,406 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:03,408 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78e5dcf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,409 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:03,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:03,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:03,413 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:03,416 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T03:40:03,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is fdc0a393186c,44285,1732160396080 2024-11-21T03:40:03,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@10ab891d 2024-11-21T03:40:03,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T03:40:03,421 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T03:40:03,425 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@577d2795, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,425 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:03,426 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:03,427 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:03,427 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:03,427 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:03,428 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14496fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,428 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:03,428 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:03,429 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:03,430 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48672, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f4e6a07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bba02c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,432 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:03,432 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:03,434 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:03,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:03,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:03,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7290076, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:03,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:03,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:03,437 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48688, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:03,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b17c100, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@113d617f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,439 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:03,439 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:03,440 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:03,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:03,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:03,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e3687d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:03,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:03,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:03,443 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:03,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b383f3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2754d9b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,445 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:03,445 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:03,447 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:03,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:03,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:03,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@175526a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:03,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:03,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:03,450 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48720, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:03,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ede423c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57135afb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,452 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:03,452 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:03,453 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:03,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:03,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:03,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9a6bf59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:03,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:03,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:03,456 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48738, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:03,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54c50a81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,459 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a48b566, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,459 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:03,459 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:03,460 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:03,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:03,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7af5f6a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:03,463 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48756, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:03,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15a3e301, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14d81b23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,465 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:03,466 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:03,467 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:03,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18ee4412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:03,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:03,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:03,469 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:03,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c3b3fdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,471 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a046d27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,471 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:03,471 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:03,472 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:03,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:03,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:03,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6e851d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:03,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:03,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:03,475 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:03,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29434e2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,477 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47ef02a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,477 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:03,477 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:03,478 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:03,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:03,478 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:03,479 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68732d4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,479 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:03,479 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:03,479 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:03,481 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:03,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a7b74c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:03,485 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:03,486 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:03,486 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:03,486 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:03,487 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:40:03,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:03,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:03,488 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:03,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:03,488 DEBUG [Thread-156 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:03,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:03,488 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:03,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:03,488 DEBUG [Thread-156 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:03,489 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:03,489 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:03,490 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:03,490 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:03,490 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:03,490 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:03,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:03,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:03,491 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:03,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:03,493 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:03,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60822, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:03,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:03,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:03,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-21T03:40:03,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:03,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:03,497 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:03,497 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:03,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:03,498 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:40:03,499 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:03,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:40:03,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T03:40:03,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:40:03,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:40:03,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:03,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:03,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:40:03,504 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:40:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:40:03,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:40:03,506 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:40:03,507 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:03,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:40:03,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:40:03,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:40:03,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:03,586 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:03,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:40:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T03:40:03,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T03:40:03,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:03,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:03,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:03,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:03,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:03,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:03,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/365d827611094d0d881dea4b7dd607ad is 50, key is test_row_0/A:col10/1732160403570/Put/seqid=0 2024-11-21T03:40:03,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741837_1013 (size=14341) 2024-11-21T03:40:03,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/365d827611094d0d881dea4b7dd607ad 2024-11-21T03:40:03,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160463723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:03,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160463745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:03,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:03,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:03,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:03,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:03,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160463745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:03,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:03,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160463749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160463752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:03,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T03:40:03,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/d70061ed7b8e45e294b1b68c0e28ac2d is 50, key is test_row_0/B:col10/1732160403570/Put/seqid=0 2024-11-21T03:40:03,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741838_1014 (size=12001) 2024-11-21T03:40:03,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/d70061ed7b8e45e294b1b68c0e28ac2d 2024-11-21T03:40:03,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T03:40:03,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:03,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:03,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:03,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:03,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:03,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:03,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160463868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:03,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:03,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:03,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:03,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:03,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160463869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:03,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:03,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160463869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:03,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:03,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:03,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:03,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:03,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160463869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:03,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:03,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160463870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:03,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:03,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:03,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/5a58e705027b4a5da9727adc20fe6541 is 50, key is test_row_0/C:col10/1732160403570/Put/seqid=0 2024-11-21T03:40:03,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741839_1015 (size=12001) 2024-11-21T03:40:03,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/5a58e705027b4a5da9727adc20fe6541 2024-11-21T03:40:03,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/365d827611094d0d881dea4b7dd607ad as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/365d827611094d0d881dea4b7dd607ad 2024-11-21T03:40:03,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/365d827611094d0d881dea4b7dd607ad, entries=200, sequenceid=14, filesize=14.0 K 2024-11-21T03:40:03,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/d70061ed7b8e45e294b1b68c0e28ac2d as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/d70061ed7b8e45e294b1b68c0e28ac2d 2024-11-21T03:40:03,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/d70061ed7b8e45e294b1b68c0e28ac2d, entries=150, sequenceid=14, filesize=11.7 K 2024-11-21T03:40:03,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/5a58e705027b4a5da9727adc20fe6541 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5a58e705027b4a5da9727adc20fe6541 2024-11-21T03:40:03,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5a58e705027b4a5da9727adc20fe6541, entries=150, sequenceid=14, filesize=11.7 K 2024-11-21T03:40:04,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 30740eafbeb6526d1c29b0b1fec56da6 in 413ms, sequenceid=14, compaction requested=false 2024-11-21T03:40:04,007 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-21T03:40:04,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:04,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T03:40:04,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:04,032 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T03:40:04,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/398b337a12014e25ae5779af4272e34a is 50, key is test_row_0/A:col10/1732160403744/Put/seqid=0 2024-11-21T03:40:04,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:04,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:04,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741840_1016 (size=12001) 2024-11-21T03:40:04,117 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/398b337a12014e25ae5779af4272e34a 2024-11-21T03:40:04,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T03:40:04,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:04,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160464120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:04,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:04,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:04,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160464125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:04,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:04,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:04,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160464132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:04,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:04,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:04,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160464132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:04,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:04,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160464140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:04,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/8e8dccb3c25d4c5d8e09c07a654258c0 is 50, key is test_row_0/B:col10/1732160403744/Put/seqid=0 2024-11-21T03:40:04,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:04,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:04,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741841_1017 (size=12001) 2024-11-21T03:40:04,215 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/8e8dccb3c25d4c5d8e09c07a654258c0 2024-11-21T03:40:04,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/1bb1422771a54839ba7ad2f83b1bb0e1 is 50, key is test_row_0/C:col10/1732160403744/Put/seqid=0 2024-11-21T03:40:04,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:04,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160464248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:04,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:04,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:04,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160464248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:04,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:04,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:04,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160464260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:04,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:04,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:04,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160464261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:04,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:04,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160464261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:04,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:04,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:04,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741842_1018 (size=12001) 2024-11-21T03:40:04,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:04,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160464467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:04,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:04,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:04,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160464468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:04,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:04,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:04,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160464478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:04,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:04,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:04,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160464478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:04,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160464478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:04,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:04,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T03:40:04,696 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/1bb1422771a54839ba7ad2f83b1bb0e1 2024-11-21T03:40:04,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/398b337a12014e25ae5779af4272e34a as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/398b337a12014e25ae5779af4272e34a 2024-11-21T03:40:04,731 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/398b337a12014e25ae5779af4272e34a, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T03:40:04,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/8e8dccb3c25d4c5d8e09c07a654258c0 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/8e8dccb3c25d4c5d8e09c07a654258c0 2024-11-21T03:40:04,753 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/8e8dccb3c25d4c5d8e09c07a654258c0, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T03:40:04,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/1bb1422771a54839ba7ad2f83b1bb0e1 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/1bb1422771a54839ba7ad2f83b1bb0e1 2024-11-21T03:40:04,774 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/1bb1422771a54839ba7ad2f83b1bb0e1, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T03:40:04,777 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 30740eafbeb6526d1c29b0b1fec56da6 in 745ms, sequenceid=37, compaction requested=false 2024-11-21T03:40:04,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:04,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:04,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-21T03:40:04,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-21T03:40:04,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-21T03:40:04,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2760 sec 2024-11-21T03:40:04,793 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.3000 sec 2024-11-21T03:40:04,794 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:04,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T03:40:04,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:04,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/34491c071a41460abea92d377d14fcea is 50, key is test_row_0/A:col10/1732160404126/Put/seqid=0 2024-11-21T03:40:04,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741843_1019 (size=14341) 2024-11-21T03:40:04,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/34491c071a41460abea92d377d14fcea 2024-11-21T03:40:04,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:04,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160464870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:04,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/755b234385d0459880f7ca266992feab is 50, key is test_row_0/B:col10/1732160404126/Put/seqid=0 2024-11-21T03:40:04,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:04,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:04,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160464875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:04,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:04,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:04,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160464875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:04,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:04,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:04,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160464881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:04,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:04,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:04,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160464882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:04,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:04,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741844_1020 (size=12001) 2024-11-21T03:40:04,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/755b234385d0459880f7ca266992feab 2024-11-21T03:40:04,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/b350a51564a64d6dad5ea6fcd9cd5c00 is 50, key is test_row_0/C:col10/1732160404126/Put/seqid=0 2024-11-21T03:40:04,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741845_1021 (size=12001) 2024-11-21T03:40:04,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/b350a51564a64d6dad5ea6fcd9cd5c00 2024-11-21T03:40:04,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/34491c071a41460abea92d377d14fcea as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/34491c071a41460abea92d377d14fcea 2024-11-21T03:40:04,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/34491c071a41460abea92d377d14fcea, entries=200, sequenceid=51, filesize=14.0 K 2024-11-21T03:40:04,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/755b234385d0459880f7ca266992feab as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/755b234385d0459880f7ca266992feab 2024-11-21T03:40:04,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:04,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160464988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:04,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:04,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:05,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:05,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160464997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:05,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:05,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:05,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160464998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:05,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:05,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160464998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:05,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:05,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,005 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:05,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160464998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:05,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/755b234385d0459880f7ca266992feab, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T03:40:05,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:05,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/b350a51564a64d6dad5ea6fcd9cd5c00 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/b350a51564a64d6dad5ea6fcd9cd5c00 2024-11-21T03:40:05,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/b350a51564a64d6dad5ea6fcd9cd5c00, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T03:40:05,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 30740eafbeb6526d1c29b0b1fec56da6 in 235ms, sequenceid=51, compaction requested=true 2024-11-21T03:40:05,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:05,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:40:05,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:05,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:40:05,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:05,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:40:05,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:05,039 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:05,039 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:05,043 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:05,052 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/B is initiating minor compaction (all files) 2024-11-21T03:40:05,053 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/B in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:05,053 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/d70061ed7b8e45e294b1b68c0e28ac2d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/8e8dccb3c25d4c5d8e09c07a654258c0, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/755b234385d0459880f7ca266992feab] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=35.2 K 2024-11-21T03:40:05,054 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:05,055 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/A is initiating minor compaction (all files) 2024-11-21T03:40:05,055 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/A in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:05,055 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/365d827611094d0d881dea4b7dd607ad, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/398b337a12014e25ae5779af4272e34a, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/34491c071a41460abea92d377d14fcea] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=39.7 K 2024-11-21T03:40:05,055 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting d70061ed7b8e45e294b1b68c0e28ac2d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732160403570 2024-11-21T03:40:05,056 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 365d827611094d0d881dea4b7dd607ad, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732160403570 2024-11-21T03:40:05,056 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e8dccb3c25d4c5d8e09c07a654258c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732160403737 2024-11-21T03:40:05,057 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 755b234385d0459880f7ca266992feab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732160404126 2024-11-21T03:40:05,059 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 398b337a12014e25ae5779af4272e34a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732160403737 2024-11-21T03:40:05,063 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34491c071a41460abea92d377d14fcea, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732160404100 2024-11-21T03:40:05,128 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#A#compaction#10 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:05,128 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#B#compaction#9 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:05,129 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/cf08d99cccab47bb816041b251243952 is 50, key is test_row_0/B:col10/1732160404126/Put/seqid=0 2024-11-21T03:40:05,129 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/323c6912a05c45fd9f9089388d4f11a3 is 50, key is test_row_0/A:col10/1732160404126/Put/seqid=0 2024-11-21T03:40:05,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741847_1023 (size=12104) 2024-11-21T03:40:05,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741846_1022 (size=12104) 2024-11-21T03:40:05,172 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/323c6912a05c45fd9f9089388d4f11a3 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/323c6912a05c45fd9f9089388d4f11a3 2024-11-21T03:40:05,202 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/A of 30740eafbeb6526d1c29b0b1fec56da6 into 323c6912a05c45fd9f9089388d4f11a3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:05,203 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:05,207 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/A, priority=13, startTime=1732160405032; duration=0sec 2024-11-21T03:40:05,207 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:05,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T03:40:05,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:05,209 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:05,209 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:A 2024-11-21T03:40:05,209 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:05,213 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:05,213 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/C is initiating minor compaction (all files) 2024-11-21T03:40:05,213 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/C in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:05,213 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5a58e705027b4a5da9727adc20fe6541, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/1bb1422771a54839ba7ad2f83b1bb0e1, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/b350a51564a64d6dad5ea6fcd9cd5c00] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=35.2 K 2024-11-21T03:40:05,214 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a58e705027b4a5da9727adc20fe6541, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732160403570 2024-11-21T03:40:05,217 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bb1422771a54839ba7ad2f83b1bb0e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732160403737 2024-11-21T03:40:05,221 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting b350a51564a64d6dad5ea6fcd9cd5c00, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732160404126 2024-11-21T03:40:05,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/ddd4610b0f7b4ce0a84f9470b5016587 is 50, key is test_row_0/A:col10/1732160404870/Put/seqid=0 2024-11-21T03:40:05,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741848_1024 (size=14341) 2024-11-21T03:40:05,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/ddd4610b0f7b4ce0a84f9470b5016587 2024-11-21T03:40:05,258 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#C#compaction#12 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:05,259 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/cc963af1f83b4b13b58d4e2347a4252c is 50, key is test_row_0/C:col10/1732160404126/Put/seqid=0 2024-11-21T03:40:05,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741849_1025 (size=12104) 2024-11-21T03:40:05,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/5fc193f8f23545de9502092eabef0b09 is 50, key is test_row_0/B:col10/1732160404870/Put/seqid=0 2024-11-21T03:40:05,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741850_1026 (size=12001) 2024-11-21T03:40:05,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160465298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:05,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160465309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:05,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160465313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:05,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:05,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:05,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160465319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:05,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:05,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160465319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:05,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:05,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160465428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:05,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:05,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:05,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160465429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:05,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:05,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160465429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:05,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:05,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:05,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:05,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160465438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:05,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:05,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:05,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160465439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:05,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:05,567 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/cf08d99cccab47bb816041b251243952 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/cf08d99cccab47bb816041b251243952 2024-11-21T03:40:05,584 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/B of 30740eafbeb6526d1c29b0b1fec56da6 into cf08d99cccab47bb816041b251243952(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:05,584 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:05,584 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/B, priority=13, startTime=1732160405038; duration=0sec 2024-11-21T03:40:05,585 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:05,585 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:B 2024-11-21T03:40:05,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:05,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160465638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160465638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:05,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:05,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:05,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160465638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:05,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T03:40:05,649 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:05,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:05,650 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:40:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160465649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-21T03:40:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T03:40:05,653 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:40:05,655 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:40:05,655 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:40:05,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160465659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:05,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:05,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:05,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/5fc193f8f23545de9502092eabef0b09 2024-11-21T03:40:05,690 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/cc963af1f83b4b13b58d4e2347a4252c as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/cc963af1f83b4b13b58d4e2347a4252c 2024-11-21T03:40:05,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/685674586cb54b9c9ec8d71ca8e98c68 is 50, key is test_row_0/C:col10/1732160404870/Put/seqid=0 2024-11-21T03:40:05,708 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/C of 30740eafbeb6526d1c29b0b1fec56da6 into cc963af1f83b4b13b58d4e2347a4252c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:05,708 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:05,709 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/C, priority=13, startTime=1732160405039; duration=0sec 2024-11-21T03:40:05,709 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:05,709 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:C 2024-11-21T03:40:05,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741851_1027 (size=12001) 2024-11-21T03:40:05,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/685674586cb54b9c9ec8d71ca8e98c68 2024-11-21T03:40:05,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T03:40:05,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/ddd4610b0f7b4ce0a84f9470b5016587 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/ddd4610b0f7b4ce0a84f9470b5016587 2024-11-21T03:40:05,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/ddd4610b0f7b4ce0a84f9470b5016587, entries=200, sequenceid=75, filesize=14.0 K 2024-11-21T03:40:05,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/5fc193f8f23545de9502092eabef0b09 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/5fc193f8f23545de9502092eabef0b09 2024-11-21T03:40:05,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/5fc193f8f23545de9502092eabef0b09, entries=150, sequenceid=75, filesize=11.7 K 2024-11-21T03:40:05,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/685674586cb54b9c9ec8d71ca8e98c68 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/685674586cb54b9c9ec8d71ca8e98c68 2024-11-21T03:40:05,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/685674586cb54b9c9ec8d71ca8e98c68, entries=150, sequenceid=75, filesize=11.7 K 2024-11-21T03:40:05,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T03:40:05,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:05,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:05,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:05,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:05,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:05,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 30740eafbeb6526d1c29b0b1fec56da6 in 604ms, sequenceid=75, compaction requested=false 2024-11-21T03:40:05,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:05,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:05,951 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:05,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T03:40:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:05,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/cdf1a1eee8d645e79a625d341234ff9c is 50, key is test_row_0/A:col10/1732160405313/Put/seqid=0 2024-11-21T03:40:05,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T03:40:05,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:05,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:05,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:05,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:05,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T03:40:05,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741852_1028 (size=12001) 2024-11-21T03:40:05,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/cdf1a1eee8d645e79a625d341234ff9c 2024-11-21T03:40:06,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/beeaf37cd645479fa79c3e31c9e97c6c is 50, key is test_row_0/B:col10/1732160405313/Put/seqid=0 2024-11-21T03:40:06,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160466012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:06,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:06,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160466014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:06,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160466018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:06,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160466018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:06,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:06,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:06,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:06,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:06,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160466020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:06,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:06,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741853_1029 (size=12001) 2024-11-21T03:40:06,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/beeaf37cd645479fa79c3e31c9e97c6c 2024-11-21T03:40:06,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/be17df8fa86f4ed980fa5c769653dc26 is 50, key is test_row_0/C:col10/1732160405313/Put/seqid=0 2024-11-21T03:40:06,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741854_1030 (size=12001) 2024-11-21T03:40:06,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T03:40:06,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:06,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:06,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:06,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:06,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:06,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160466129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:06,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:06,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160466129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:06,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:06,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160466129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:06,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:06,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160466130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:06,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:06,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160466130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:06,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:06,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:06,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T03:40:06,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T03:40:06,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:06,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:06,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:06,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:06,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:06,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:06,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:06,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160466338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160466338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:06,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160466338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:06,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:06,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160466340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:06,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:06,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:06,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160466349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:06,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:06,354 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T03:40:06,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T03:40:06,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:06,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:06,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:06,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:06,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:06,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:06,489 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-21T03:40:06,491 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-21T03:40:06,492 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-21T03:40:06,494 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T03:40:06,494 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-21T03:40:06,494 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-21T03:40:06,494 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-21T03:40:06,496 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-21T03:40:06,496 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-21T03:40:06,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/be17df8fa86f4ed980fa5c769653dc26 2024-11-21T03:40:06,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/cdf1a1eee8d645e79a625d341234ff9c as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/cdf1a1eee8d645e79a625d341234ff9c 2024-11-21T03:40:06,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/cdf1a1eee8d645e79a625d341234ff9c, entries=150, sequenceid=92, filesize=11.7 K 2024-11-21T03:40:06,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/beeaf37cd645479fa79c3e31c9e97c6c as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/beeaf37cd645479fa79c3e31c9e97c6c 2024-11-21T03:40:06,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/beeaf37cd645479fa79c3e31c9e97c6c, entries=150, sequenceid=92, filesize=11.7 K 2024-11-21T03:40:06,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/be17df8fa86f4ed980fa5c769653dc26 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/be17df8fa86f4ed980fa5c769653dc26 2024-11-21T03:40:06,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/be17df8fa86f4ed980fa5c769653dc26, entries=150, sequenceid=92, filesize=11.7 K 2024-11-21T03:40:06,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T03:40:06,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:06,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:06,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:06,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:06,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:06,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 30740eafbeb6526d1c29b0b1fec56da6 in 642ms, sequenceid=92, compaction requested=true 2024-11-21T03:40:06,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:06,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:40:06,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:06,594 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:06,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:06,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:40:06,594 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:06,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:06,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:40:06,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:06,596 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:06,596 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/A is initiating minor compaction (all files) 2024-11-21T03:40:06,596 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:06,596 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/A in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:06,596 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/B is initiating minor compaction (all files) 2024-11-21T03:40:06,596 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/B in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:06,597 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/323c6912a05c45fd9f9089388d4f11a3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/ddd4610b0f7b4ce0a84f9470b5016587, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/cdf1a1eee8d645e79a625d341234ff9c] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=37.5 K 2024-11-21T03:40:06,597 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/cf08d99cccab47bb816041b251243952, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/5fc193f8f23545de9502092eabef0b09, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/beeaf37cd645479fa79c3e31c9e97c6c] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=35.3 K 2024-11-21T03:40:06,598 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting cf08d99cccab47bb816041b251243952, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732160404126 2024-11-21T03:40:06,600 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 323c6912a05c45fd9f9089388d4f11a3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732160404126 2024-11-21T03:40:06,600 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fc193f8f23545de9502092eabef0b09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732160404870 2024-11-21T03:40:06,601 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting beeaf37cd645479fa79c3e31c9e97c6c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732160405312 2024-11-21T03:40:06,602 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddd4610b0f7b4ce0a84f9470b5016587, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732160404870 2024-11-21T03:40:06,604 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdf1a1eee8d645e79a625d341234ff9c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732160405312 2024-11-21T03:40:06,627 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#B#compaction#18 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:06,628 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/7caa146aab8c4aa9b527e2fb27e13932 is 50, key is test_row_0/B:col10/1732160405313/Put/seqid=0 2024-11-21T03:40:06,637 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#A#compaction#19 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:06,638 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/044b2a01daaf4df0a374bdf160b256a8 is 50, key is test_row_0/A:col10/1732160405313/Put/seqid=0 2024-11-21T03:40:06,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:06,662 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:06,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T03:40:06,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741855_1031 (size=12207) 2024-11-21T03:40:06,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741856_1032 (size=12207) 2024-11-21T03:40:06,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/f059dcb9a100447ba39d447788c7f0cd is 50, key is test_row_0/A:col10/1732160406013/Put/seqid=0 2024-11-21T03:40:06,696 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/044b2a01daaf4df0a374bdf160b256a8 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/044b2a01daaf4df0a374bdf160b256a8 2024-11-21T03:40:06,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:06,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160466698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:06,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:06,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:06,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160466699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:06,716 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/A of 30740eafbeb6526d1c29b0b1fec56da6 into 044b2a01daaf4df0a374bdf160b256a8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:06,716 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:06,716 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/A, priority=13, startTime=1732160406594; duration=0sec 2024-11-21T03:40:06,716 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:06,717 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:A 2024-11-21T03:40:06,717 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:06,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160466709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:06,719 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:06,720 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/C is initiating minor compaction (all files) 2024-11-21T03:40:06,720 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/C in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:06,720 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/cc963af1f83b4b13b58d4e2347a4252c, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/685674586cb54b9c9ec8d71ca8e98c68, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/be17df8fa86f4ed980fa5c769653dc26] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=35.3 K 2024-11-21T03:40:06,721 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc963af1f83b4b13b58d4e2347a4252c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732160404126 2024-11-21T03:40:06,723 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 685674586cb54b9c9ec8d71ca8e98c68, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1732160404870 2024-11-21T03:40:06,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:06,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160466712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:06,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:06,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160466713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:06,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,725 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting be17df8fa86f4ed980fa5c769653dc26, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732160405312 2024-11-21T03:40:06,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,725 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:06,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,725 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:06,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741857_1033 (size=16681) 2024-11-21T03:40:06,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/f059dcb9a100447ba39d447788c7f0cd 2024-11-21T03:40:06,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T03:40:06,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:06,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:06,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:06,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:06,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:06,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:06,770 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#C#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:06,771 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/5b0142154e2c44a2a048d4115ee2a7a0 is 50, key is test_row_0/C:col10/1732160405313/Put/seqid=0 2024-11-21T03:40:06,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/f54799c1f0594f4e9d23473d38743232 is 50, key is test_row_0/B:col10/1732160406013/Put/seqid=0 2024-11-21T03:40:06,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T03:40:06,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741858_1034 (size=12207) 2024-11-21T03:40:06,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:06,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160466817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:06,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:06,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160466817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:06,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:06,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160466828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:06,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:06,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:06,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160466828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:06,837 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/5b0142154e2c44a2a048d4115ee2a7a0 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5b0142154e2c44a2a048d4115ee2a7a0 2024-11-21T03:40:06,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:06,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:06,849 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/C of 30740eafbeb6526d1c29b0b1fec56da6 into 5b0142154e2c44a2a048d4115ee2a7a0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:06,849 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:06,850 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/C, priority=13, startTime=1732160406594; duration=0sec 2024-11-21T03:40:06,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160466837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:06,850 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:06,850 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:C 2024-11-21T03:40:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:06,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741859_1035 (size=12001) 2024-11-21T03:40:06,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T03:40:06,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:06,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:06,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:06,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:06,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:06,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:07,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:07,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160467030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:07,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:07,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:07,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:07,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160467038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:07,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:07,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:07,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:07,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:07,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160467048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:07,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160467048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:07,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160467057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:07,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T03:40:07,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:07,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:07,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:07,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:07,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:07,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:07,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:07,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:07,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:07,088 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/7caa146aab8c4aa9b527e2fb27e13932 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/7caa146aab8c4aa9b527e2fb27e13932 2024-11-21T03:40:07,102 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/B of 30740eafbeb6526d1c29b0b1fec56da6 into 7caa146aab8c4aa9b527e2fb27e13932(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:07,102 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:07,103 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/B, priority=13, startTime=1732160406594; duration=0sec 2024-11-21T03:40:07,104 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:07,104 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:B 2024-11-21T03:40:07,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T03:40:07,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:07,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:07,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:07,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:07,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:07,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:07,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/f54799c1f0594f4e9d23473d38743232 2024-11-21T03:40:07,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/75c1e6c8a3824e5f90441398a9dd65c9 is 50, key is test_row_0/C:col10/1732160406013/Put/seqid=0 2024-11-21T03:40:07,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741860_1036 (size=12001) 2024-11-21T03:40:07,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/75c1e6c8a3824e5f90441398a9dd65c9 2024-11-21T03:40:07,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/f059dcb9a100447ba39d447788c7f0cd as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/f059dcb9a100447ba39d447788c7f0cd 2024-11-21T03:40:07,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:07,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160467337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:07,343 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:07,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:07,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:07,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/f059dcb9a100447ba39d447788c7f0cd, entries=250, sequenceid=115, filesize=16.3 K 2024-11-21T03:40:07,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/f54799c1f0594f4e9d23473d38743232 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/f54799c1f0594f4e9d23473d38743232 2024-11-21T03:40:07,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/f54799c1f0594f4e9d23473d38743232, entries=150, sequenceid=115, filesize=11.7 K 2024-11-21T03:40:07,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/75c1e6c8a3824e5f90441398a9dd65c9 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/75c1e6c8a3824e5f90441398a9dd65c9 2024-11-21T03:40:07,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160467357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:07,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:07,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160467357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:07,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:07,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:07,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:07,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:07,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:07,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:07,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160467367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:07,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:07,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:07,375 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:07,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:07,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160467367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:07,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/75c1e6c8a3824e5f90441398a9dd65c9, entries=150, sequenceid=115, filesize=11.7 K 2024-11-21T03:40:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:07,376 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:07,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 30740eafbeb6526d1c29b0b1fec56da6 in 715ms, sequenceid=115, compaction requested=false 2024-11-21T03:40:07,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:07,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T03:40:07,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:07,379 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T03:40:07,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/428fa94941b1435e85b6e2528e875336 is 50, key is test_row_0/A:col10/1732160406703/Put/seqid=0 2024-11-21T03:40:07,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741861_1037 (size=12001) 2024-11-21T03:40:07,474 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/428fa94941b1435e85b6e2528e875336 2024-11-21T03:40:07,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/14a3648a305c4737a9b4e82c55cd7891 is 50, key is test_row_0/B:col10/1732160406703/Put/seqid=0 2024-11-21T03:40:07,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741862_1038 (size=12001) 2024-11-21T03:40:07,508 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/14a3648a305c4737a9b4e82c55cd7891 2024-11-21T03:40:07,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/5736edc2061b44d1a63ab1013ce2274d is 50, key is test_row_0/C:col10/1732160406703/Put/seqid=0 2024-11-21T03:40:07,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741863_1039 (size=12001) 2024-11-21T03:40:07,569 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/5736edc2061b44d1a63ab1013ce2274d 2024-11-21T03:40:07,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/428fa94941b1435e85b6e2528e875336 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/428fa94941b1435e85b6e2528e875336 2024-11-21T03:40:07,591 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/428fa94941b1435e85b6e2528e875336, entries=150, sequenceid=132, filesize=11.7 K 2024-11-21T03:40:07,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/14a3648a305c4737a9b4e82c55cd7891 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/14a3648a305c4737a9b4e82c55cd7891 2024-11-21T03:40:07,607 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/14a3648a305c4737a9b4e82c55cd7891, entries=150, sequenceid=132, filesize=11.7 K 2024-11-21T03:40:07,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/5736edc2061b44d1a63ab1013ce2274d as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5736edc2061b44d1a63ab1013ce2274d 2024-11-21T03:40:07,619 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5736edc2061b44d1a63ab1013ce2274d, entries=150, sequenceid=132, filesize=11.7 K 2024-11-21T03:40:07,621 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for 30740eafbeb6526d1c29b0b1fec56da6 in 242ms, sequenceid=132, compaction requested=true 2024-11-21T03:40:07,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:07,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:07,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-21T03:40:07,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-21T03:40:07,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-21T03:40:07,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9730 sec 2024-11-21T03:40:07,641 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.9880 sec 2024-11-21T03:40:07,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T03:40:07,798 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:07,799 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:40:07,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-21T03:40:07,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T03:40:07,802 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:40:07,804 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:40:07,804 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:40:07,901 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:07,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:40:07,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:07,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T03:40:07,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/46e61b69e4874d19a005b0f5528a98b1 is 50, key is test_row_0/A:col10/1732160407892/Put/seqid=0 2024-11-21T03:40:07,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741864_1040 (size=14541) 2024-11-21T03:40:07,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/46e61b69e4874d19a005b0f5528a98b1 2024-11-21T03:40:07,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T03:40:07,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:07,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:07,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:07,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:07,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:07,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:07,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/aeba7022284546979ae9f9dcee48c520 is 50, key is test_row_0/B:col10/1732160407892/Put/seqid=0 2024-11-21T03:40:08,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:08,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160467984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:08,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160467985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:08,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,006 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:08,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:08,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160467987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:08,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:08,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160467999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,008 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:08,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:08,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160468003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:08,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741865_1041 (size=12151) 2024-11-21T03:40:08,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:08,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160468107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:08,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:08,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T03:40:08,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:08,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160468118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:08,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:08,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160468118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:08,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160468119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:08,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:08,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:08,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160468119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:08,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T03:40:08,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,125 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:08,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:08,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:08,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:08,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:08,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:08,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:08,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T03:40:08,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:08,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:08,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:08,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:08,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:08,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:08,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160468319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:08,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:08,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160468327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:08,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:08,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160468327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:08,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:08,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160468327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:08,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:08,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:08,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:08,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:08,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160468337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:08,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:08,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T03:40:08,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T03:40:08,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:08,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:08,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:08,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:08,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:08,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:08,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/aeba7022284546979ae9f9dcee48c520 2024-11-21T03:40:08,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/6137c643a72549f4bd00b1066cb8e1e3 is 50, key is test_row_0/C:col10/1732160407892/Put/seqid=0 2024-11-21T03:40:08,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741866_1042 (size=12151) 2024-11-21T03:40:08,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T03:40:08,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:08,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:08,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:08,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:08,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:08,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:08,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:08,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160468627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:08,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:08,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:08,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160468637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:08,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:08,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:08,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:08,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160468647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:08,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160468647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:08,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:08,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160468647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:08,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T03:40:08,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:08,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:08,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:08,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:08,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:08,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:08,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T03:40:08,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:08,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:08,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:08,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:08,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:08,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/6137c643a72549f4bd00b1066cb8e1e3 2024-11-21T03:40:08,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/46e61b69e4874d19a005b0f5528a98b1 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/46e61b69e4874d19a005b0f5528a98b1 2024-11-21T03:40:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T03:40:08,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/46e61b69e4874d19a005b0f5528a98b1, entries=200, sequenceid=143, filesize=14.2 K 2024-11-21T03:40:08,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/aeba7022284546979ae9f9dcee48c520 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/aeba7022284546979ae9f9dcee48c520 2024-11-21T03:40:08,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/aeba7022284546979ae9f9dcee48c520, entries=150, sequenceid=143, filesize=11.9 K 2024-11-21T03:40:08,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/6137c643a72549f4bd00b1066cb8e1e3 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/6137c643a72549f4bd00b1066cb8e1e3 2024-11-21T03:40:08,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/6137c643a72549f4bd00b1066cb8e1e3, entries=150, sequenceid=143, filesize=11.9 K 2024-11-21T03:40:08,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 30740eafbeb6526d1c29b0b1fec56da6 in 1080ms, sequenceid=143, compaction requested=true 2024-11-21T03:40:08,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:08,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:40:08,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:08,982 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:40:08,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:40:08,982 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:40:08,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:08,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:40:08,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:08,985 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55430 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:40:08,985 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:40:08,985 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/A is initiating minor compaction (all files) 2024-11-21T03:40:08,985 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/B is initiating minor compaction (all files) 2024-11-21T03:40:08,985 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/B in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:08,985 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/A in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:08,986 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/7caa146aab8c4aa9b527e2fb27e13932, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/f54799c1f0594f4e9d23473d38743232, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/14a3648a305c4737a9b4e82c55cd7891, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/aeba7022284546979ae9f9dcee48c520] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=47.2 K 2024-11-21T03:40:08,986 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/044b2a01daaf4df0a374bdf160b256a8, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/f059dcb9a100447ba39d447788c7f0cd, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/428fa94941b1435e85b6e2528e875336, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/46e61b69e4874d19a005b0f5528a98b1] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=54.1 K 2024-11-21T03:40:08,987 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 7caa146aab8c4aa9b527e2fb27e13932, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732160405312 2024-11-21T03:40:08,987 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 044b2a01daaf4df0a374bdf160b256a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732160405312 2024-11-21T03:40:08,987 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting f059dcb9a100447ba39d447788c7f0cd, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732160405999 2024-11-21T03:40:08,988 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting f54799c1f0594f4e9d23473d38743232, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732160406013 2024-11-21T03:40:08,988 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 428fa94941b1435e85b6e2528e875336, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732160406682 2024-11-21T03:40:08,989 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 14a3648a305c4737a9b4e82c55cd7891, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732160406682 2024-11-21T03:40:08,989 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46e61b69e4874d19a005b0f5528a98b1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732160407885 2024-11-21T03:40:08,990 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting aeba7022284546979ae9f9dcee48c520, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732160407885 2024-11-21T03:40:09,030 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#B#compaction#30 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:09,031 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/3b3f9be376cb42fdbc94865993799577 is 50, key is test_row_0/B:col10/1732160407892/Put/seqid=0 2024-11-21T03:40:09,045 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#A#compaction#31 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:09,046 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/e34a2b1612b14989b52791c23720b096 is 50, key is test_row_0/A:col10/1732160407892/Put/seqid=0 2024-11-21T03:40:09,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T03:40:09,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:09,061 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T03:40:09,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/ef82e13423da4644ad1a73a1bd825e8e is 50, key is test_row_0/A:col10/1732160407981/Put/seqid=0 2024-11-21T03:40:09,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741867_1043 (size=12493) 2024-11-21T03:40:09,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741868_1044 (size=12493) 2024-11-21T03:40:09,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:09,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741869_1045 (size=12151) 2024-11-21T03:40:09,182 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/ef82e13423da4644ad1a73a1bd825e8e 2024-11-21T03:40:09,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:09,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160469186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:09,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,208 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:09,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160469189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:09,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:09,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160469189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:09,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:09,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160469200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:09,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:09,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/96f77690e726435db394d700776af692 is 50, key is test_row_0/B:col10/1732160407981/Put/seqid=0 2024-11-21T03:40:09,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:09,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160469216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:09,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:09,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741870_1046 (size=12151) 2024-11-21T03:40:09,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160469317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:09,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160469318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:09,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:09,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160469327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:09,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,334 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:09,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160469327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:09,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160469337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:09,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:09,524 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/3b3f9be376cb42fdbc94865993799577 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/3b3f9be376cb42fdbc94865993799577 2024-11-21T03:40:09,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:09,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160469528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:09,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:09,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160469528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:09,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:09,536 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/B of 30740eafbeb6526d1c29b0b1fec56da6 into 3b3f9be376cb42fdbc94865993799577(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:09,536 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:09,536 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/B, priority=12, startTime=1732160408982; duration=0sec 2024-11-21T03:40:09,537 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:09,537 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:B 2024-11-21T03:40:09,537 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:40:09,540 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:40:09,541 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/C is initiating minor compaction (all files) 2024-11-21T03:40:09,541 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/C in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:09,541 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5b0142154e2c44a2a048d4115ee2a7a0, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/75c1e6c8a3824e5f90441398a9dd65c9, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5736edc2061b44d1a63ab1013ce2274d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/6137c643a72549f4bd00b1066cb8e1e3] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=47.2 K 2024-11-21T03:40:09,542 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b0142154e2c44a2a048d4115ee2a7a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732160405312 2024-11-21T03:40:09,543 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 75c1e6c8a3824e5f90441398a9dd65c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1732160406013 2024-11-21T03:40:09,550 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/e34a2b1612b14989b52791c23720b096 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/e34a2b1612b14989b52791c23720b096 2024-11-21T03:40:09,551 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 5736edc2061b44d1a63ab1013ce2274d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732160406682 2024-11-21T03:40:09,552 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 6137c643a72549f4bd00b1066cb8e1e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732160407885 2024-11-21T03:40:09,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:09,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160469538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:09,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:09,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160469547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:09,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:09,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:09,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160469547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:09,563 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/A of 30740eafbeb6526d1c29b0b1fec56da6 into e34a2b1612b14989b52791c23720b096(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:09,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,563 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:09,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,563 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/A, priority=12, startTime=1732160408982; duration=0sec 2024-11-21T03:40:09,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:09,563 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:09,565 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:A 2024-11-21T03:40:09,585 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#C#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:09,586 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/0aa9c4640b774cba9f501279ca734a9c is 50, key is test_row_0/C:col10/1732160407892/Put/seqid=0 2024-11-21T03:40:09,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741871_1047 (size=12493) 2024-11-21T03:40:09,666 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/96f77690e726435db394d700776af692 2024-11-21T03:40:09,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/bbfb949d9d30470599a79becb807878c is 50, key is test_row_0/C:col10/1732160407981/Put/seqid=0 2024-11-21T03:40:09,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741872_1048 (size=12151) 2024-11-21T03:40:09,755 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/bbfb949d9d30470599a79becb807878c 2024-11-21T03:40:09,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/ef82e13423da4644ad1a73a1bd825e8e as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/ef82e13423da4644ad1a73a1bd825e8e 2024-11-21T03:40:09,777 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/ef82e13423da4644ad1a73a1bd825e8e, entries=150, sequenceid=168, filesize=11.9 K 2024-11-21T03:40:09,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/96f77690e726435db394d700776af692 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/96f77690e726435db394d700776af692 2024-11-21T03:40:09,804 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/96f77690e726435db394d700776af692, entries=150, sequenceid=168, filesize=11.9 K 2024-11-21T03:40:09,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/bbfb949d9d30470599a79becb807878c as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/bbfb949d9d30470599a79becb807878c 2024-11-21T03:40:09,815 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/bbfb949d9d30470599a79becb807878c, entries=150, sequenceid=168, filesize=11.9 K 2024-11-21T03:40:09,817 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 30740eafbeb6526d1c29b0b1fec56da6 in 756ms, sequenceid=168, compaction requested=false 2024-11-21T03:40:09,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:09,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:09,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-21T03:40:09,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-21T03:40:09,823 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-21T03:40:09,824 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0160 sec 2024-11-21T03:40:09,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.0250 sec 2024-11-21T03:40:09,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:09,853 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:09,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T03:40:09,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/de9572f63298471890ed5b972f5baa74 is 50, key is test_row_0/A:col10/1732160409182/Put/seqid=0 2024-11-21T03:40:09,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741873_1049 (size=16931) 2024-11-21T03:40:09,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/de9572f63298471890ed5b972f5baa74 2024-11-21T03:40:09,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:09,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:09,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160469906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:09,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160469909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:09,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:09,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:09,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:09,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160469915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:09,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:09,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160469917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:09,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:09,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:09,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/8ffcf7cc8bdf41d989b42c376cd4889b is 50, key is test_row_0/B:col10/1732160409182/Put/seqid=0 2024-11-21T03:40:09,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:09,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160469917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:09,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:09,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:09,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741874_1050 (size=12151) 2024-11-21T03:40:09,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/8ffcf7cc8bdf41d989b42c376cd4889b 2024-11-21T03:40:09,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T03:40:09,958 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:09,959 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:40:09,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-21T03:40:09,964 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:40:09,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T03:40:09,966 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:40:09,966 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:40:09,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/3bd5fe1a466742959222d4a46f38dcfd is 50, key is test_row_0/C:col10/1732160409182/Put/seqid=0 2024-11-21T03:40:09,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741875_1051 (size=12151) 2024-11-21T03:40:09,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/3bd5fe1a466742959222d4a46f38dcfd 2024-11-21T03:40:09,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/de9572f63298471890ed5b972f5baa74 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/de9572f63298471890ed5b972f5baa74 2024-11-21T03:40:10,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/de9572f63298471890ed5b972f5baa74, entries=250, sequenceid=184, filesize=16.5 K 2024-11-21T03:40:10,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/8ffcf7cc8bdf41d989b42c376cd4889b as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/8ffcf7cc8bdf41d989b42c376cd4889b 2024-11-21T03:40:10,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:10,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160470027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:10,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:10,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:10,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160470028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:10,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/8ffcf7cc8bdf41d989b42c376cd4889b, entries=150, sequenceid=184, filesize=11.9 K 2024-11-21T03:40:10,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/3bd5fe1a466742959222d4a46f38dcfd as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/3bd5fe1a466742959222d4a46f38dcfd 2024-11-21T03:40:10,049 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/0aa9c4640b774cba9f501279ca734a9c as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0aa9c4640b774cba9f501279ca734a9c 2024-11-21T03:40:10,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:10,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160470038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:10,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:10,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160470038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:10,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:10,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:10,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160470038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:10,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,052 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:10,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:10,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/3bd5fe1a466742959222d4a46f38dcfd, entries=150, sequenceid=184, filesize=11.9 K 2024-11-21T03:40:10,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 30740eafbeb6526d1c29b0b1fec56da6 in 210ms, sequenceid=184, compaction requested=true 2024-11-21T03:40:10,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:10,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:A, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:40:10,064 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:10,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:10,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:B, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:40:10,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:10,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:40:10,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T03:40:10,066 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:10,066 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/A is initiating minor compaction (all files) 2024-11-21T03:40:10,066 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/A in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:10,066 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/e34a2b1612b14989b52791c23720b096, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/ef82e13423da4644ad1a73a1bd825e8e, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/de9572f63298471890ed5b972f5baa74] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=40.6 K 2024-11-21T03:40:10,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T03:40:10,067 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting e34a2b1612b14989b52791c23720b096, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732160407885 2024-11-21T03:40:10,068 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef82e13423da4644ad1a73a1bd825e8e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732160407956 2024-11-21T03:40:10,069 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting de9572f63298471890ed5b972f5baa74, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732160409182 2024-11-21T03:40:10,076 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/C of 30740eafbeb6526d1c29b0b1fec56da6 into 0aa9c4640b774cba9f501279ca734a9c(size=12.2 K), total size for store is 35.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:10,076 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:10,076 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/C, priority=12, startTime=1732160408983; duration=0sec 2024-11-21T03:40:10,077 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T03:40:10,077 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:C 2024-11-21T03:40:10,077 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:C 2024-11-21T03:40:10,077 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:10,081 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:10,082 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/B is initiating minor compaction (all files) 2024-11-21T03:40:10,082 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/B in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:10,082 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/3b3f9be376cb42fdbc94865993799577, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/96f77690e726435db394d700776af692, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/8ffcf7cc8bdf41d989b42c376cd4889b] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=35.9 K 2024-11-21T03:40:10,083 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#A#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:10,084 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/9814b5464c774bd893bc5451c4d9c911 is 50, key is test_row_0/A:col10/1732160409182/Put/seqid=0 2024-11-21T03:40:10,094 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b3f9be376cb42fdbc94865993799577, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732160407885 2024-11-21T03:40:10,096 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 96f77690e726435db394d700776af692, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732160407956 2024-11-21T03:40:10,097 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ffcf7cc8bdf41d989b42c376cd4889b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732160409182 2024-11-21T03:40:10,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T03:40:10,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:10,121 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T03:40:10,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741876_1052 (size=12595) 2024-11-21T03:40:10,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/78ae3972b89f43caaf9a7cdd51d2e94c is 50, key is test_row_0/A:col10/1732160409915/Put/seqid=0 2024-11-21T03:40:10,149 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/9814b5464c774bd893bc5451c4d9c911 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/9814b5464c774bd893bc5451c4d9c911 2024-11-21T03:40:10,152 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#B#compaction#41 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:10,153 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/ed07bd9f370543229293f566a1c2f4e4 is 50, key is test_row_0/B:col10/1732160409182/Put/seqid=0 2024-11-21T03:40:10,159 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/A of 30740eafbeb6526d1c29b0b1fec56da6 into 9814b5464c774bd893bc5451c4d9c911(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:10,160 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:10,160 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/A, priority=13, startTime=1732160410064; duration=0sec 2024-11-21T03:40:10,160 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:10,160 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:A 2024-11-21T03:40:10,160 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:10,162 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:10,163 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/C is initiating minor compaction (all files) 2024-11-21T03:40:10,163 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/C in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:10,163 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0aa9c4640b774cba9f501279ca734a9c, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/bbfb949d9d30470599a79becb807878c, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/3bd5fe1a466742959222d4a46f38dcfd] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=35.9 K 2024-11-21T03:40:10,164 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0aa9c4640b774cba9f501279ca734a9c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732160407885 2024-11-21T03:40:10,164 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbfb949d9d30470599a79becb807878c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1732160407956 2024-11-21T03:40:10,165 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bd5fe1a466742959222d4a46f38dcfd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732160409182 2024-11-21T03:40:10,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741877_1053 (size=12151) 2024-11-21T03:40:10,182 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/78ae3972b89f43caaf9a7cdd51d2e94c 2024-11-21T03:40:10,204 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#C#compaction#42 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:10,204 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/72281ac7c9f14f3b94dbcb63d84ba956 is 50, key is test_row_0/C:col10/1732160409182/Put/seqid=0 2024-11-21T03:40:10,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741878_1054 (size=12595) 2024-11-21T03:40:10,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:10,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:10,254 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/ed07bd9f370543229293f566a1c2f4e4 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/ed07bd9f370543229293f566a1c2f4e4 2024-11-21T03:40:10,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/3c219f366fc94025a61d319f970789fd is 50, key is test_row_0/B:col10/1732160409915/Put/seqid=0 2024-11-21T03:40:10,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741879_1055 (size=12595) 2024-11-21T03:40:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T03:40:10,286 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/B of 30740eafbeb6526d1c29b0b1fec56da6 into ed07bd9f370543229293f566a1c2f4e4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:10,286 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:10,286 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/B, priority=13, startTime=1732160410064; duration=0sec 2024-11-21T03:40:10,286 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:10,286 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:B 2024-11-21T03:40:10,287 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/72281ac7c9f14f3b94dbcb63d84ba956 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/72281ac7c9f14f3b94dbcb63d84ba956 2024-11-21T03:40:10,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160470276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:10,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:10,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160470290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160470290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:10,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:10,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160470294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:10,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160470294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:10,309 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/C of 30740eafbeb6526d1c29b0b1fec56da6 into 72281ac7c9f14f3b94dbcb63d84ba956(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:10,309 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:10,309 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/C, priority=13, startTime=1732160410064; duration=0sec 2024-11-21T03:40:10,309 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:10,309 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:C 2024-11-21T03:40:10,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741880_1056 (size=12151) 2024-11-21T03:40:10,326 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/3c219f366fc94025a61d319f970789fd 2024-11-21T03:40:10,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/f05845015b7b42cdb10eab617a3787d1 is 50, key is test_row_0/C:col10/1732160409915/Put/seqid=0 2024-11-21T03:40:10,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741881_1057 (size=12151) 2024-11-21T03:40:10,388 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/f05845015b7b42cdb10eab617a3787d1 2024-11-21T03:40:10,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/78ae3972b89f43caaf9a7cdd51d2e94c as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/78ae3972b89f43caaf9a7cdd51d2e94c 2024-11-21T03:40:10,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:10,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160470397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:10,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:10,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160470407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:10,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:10,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:10,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:10,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160470407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:10,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160470407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:10,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:10,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:10,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160470407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:10,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:10,430 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/78ae3972b89f43caaf9a7cdd51d2e94c, entries=150, sequenceid=208, filesize=11.9 K 2024-11-21T03:40:10,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/3c219f366fc94025a61d319f970789fd as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/3c219f366fc94025a61d319f970789fd 2024-11-21T03:40:10,441 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/3c219f366fc94025a61d319f970789fd, entries=150, sequenceid=208, filesize=11.9 K 2024-11-21T03:40:10,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/f05845015b7b42cdb10eab617a3787d1 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/f05845015b7b42cdb10eab617a3787d1 2024-11-21T03:40:10,472 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/f05845015b7b42cdb10eab617a3787d1, entries=150, sequenceid=208, filesize=11.9 K 2024-11-21T03:40:10,474 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 30740eafbeb6526d1c29b0b1fec56da6 in 354ms, sequenceid=208, compaction requested=false 2024-11-21T03:40:10,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:10,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:10,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-21T03:40:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-21T03:40:10,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-21T03:40:10,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 511 msec 2024-11-21T03:40:10,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 525 msec 2024-11-21T03:40:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T03:40:10,588 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:10,588 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:40:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-21T03:40:10,592 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:40:10,593 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:40:10,594 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:40:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T03:40:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:10,614 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:10,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T03:40:10,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/25cb959f983c435c994847ba3343a8f0 is 50, key is test_row_0/A:col10/1732160410611/Put/seqid=0 2024-11-21T03:40:10,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741882_1058 (size=14541) 2024-11-21T03:40:10,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/25cb959f983c435c994847ba3343a8f0 2024-11-21T03:40:10,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/c524b661ad2e453499abdb2fffed041a is 50, key is test_row_0/B:col10/1732160410611/Put/seqid=0 2024-11-21T03:40:10,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:10,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160470673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:10,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:10,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160470688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,697 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:10,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160470691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:10,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:10,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160470691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:10,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:10,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:10,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160470691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:10,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:10,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T03:40:10,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741883_1059 (size=12151) 2024-11-21T03:40:10,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T03:40:10,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:10,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:10,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:10,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:10,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:10,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:10,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:10,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160470797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:10,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:10,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160470807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:10,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:10,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:10,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160470808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:10,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160470808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160470807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:10,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:10,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:10,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T03:40:10,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:10,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:10,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:10,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:10,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:10,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:10,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T03:40:11,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160471007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:11,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:11,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160471017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:11,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:11,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:11,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160471017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:11,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160471027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:11,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:11,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160471027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:11,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:11,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T03:40:11,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:11,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:11,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:11,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:11,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:11,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:11,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/c524b661ad2e453499abdb2fffed041a 2024-11-21T03:40:11,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/0d57504694d44965b3c1ba3963b477d5 is 50, key is test_row_0/C:col10/1732160410611/Put/seqid=0 2024-11-21T03:40:11,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741884_1060 (size=12151) 2024-11-21T03:40:11,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/0d57504694d44965b3c1ba3963b477d5 2024-11-21T03:40:11,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/25cb959f983c435c994847ba3343a8f0 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/25cb959f983c435c994847ba3343a8f0 2024-11-21T03:40:11,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/25cb959f983c435c994847ba3343a8f0, entries=200, sequenceid=226, filesize=14.2 K 2024-11-21T03:40:11,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/c524b661ad2e453499abdb2fffed041a as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/c524b661ad2e453499abdb2fffed041a 2024-11-21T03:40:11,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/c524b661ad2e453499abdb2fffed041a, entries=150, sequenceid=226, filesize=11.9 K 2024-11-21T03:40:11,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/0d57504694d44965b3c1ba3963b477d5 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0d57504694d44965b3c1ba3963b477d5 2024-11-21T03:40:11,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T03:40:11,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:11,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:11,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:11,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:11,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:11,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0d57504694d44965b3c1ba3963b477d5, entries=150, sequenceid=226, filesize=11.9 K 2024-11-21T03:40:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:11,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 30740eafbeb6526d1c29b0b1fec56da6 in 602ms, sequenceid=226, compaction requested=true 2024-11-21T03:40:11,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:11,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:40:11,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:11,217 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:11,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:40:11,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:11,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:40:11,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:11,217 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:11,223 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:11,223 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:11,224 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/B is initiating minor compaction (all files) 2024-11-21T03:40:11,224 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/A is initiating minor compaction (all files) 2024-11-21T03:40:11,224 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/B in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:11,224 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/A in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:11,224 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/ed07bd9f370543229293f566a1c2f4e4, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/3c219f366fc94025a61d319f970789fd, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/c524b661ad2e453499abdb2fffed041a] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=36.0 K 2024-11-21T03:40:11,224 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/9814b5464c774bd893bc5451c4d9c911, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/78ae3972b89f43caaf9a7cdd51d2e94c, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/25cb959f983c435c994847ba3343a8f0] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=38.4 K 2024-11-21T03:40:11,224 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting ed07bd9f370543229293f566a1c2f4e4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732160409182 2024-11-21T03:40:11,225 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9814b5464c774bd893bc5451c4d9c911, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732160409182 2024-11-21T03:40:11,225 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c219f366fc94025a61d319f970789fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732160409902 2024-11-21T03:40:11,225 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting c524b661ad2e453499abdb2fffed041a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732160410277 2024-11-21T03:40:11,225 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78ae3972b89f43caaf9a7cdd51d2e94c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732160409902 2024-11-21T03:40:11,226 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25cb959f983c435c994847ba3343a8f0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732160410277 2024-11-21T03:40:11,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T03:40:11,257 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#B#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:11,258 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/a83f2f701e174734ad546b5b3c938a5e is 50, key is test_row_0/B:col10/1732160410611/Put/seqid=0 2024-11-21T03:40:11,268 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#A#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:11,268 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/bcbab42508504d0a881299d502f86f5b is 50, key is test_row_0/A:col10/1732160410611/Put/seqid=0 2024-11-21T03:40:11,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741885_1061 (size=12697) 2024-11-21T03:40:11,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741886_1062 (size=12697) 2024-11-21T03:40:11,301 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/a83f2f701e174734ad546b5b3c938a5e as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/a83f2f701e174734ad546b5b3c938a5e 2024-11-21T03:40:11,311 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/B of 30740eafbeb6526d1c29b0b1fec56da6 into a83f2f701e174734ad546b5b3c938a5e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:11,311 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:11,311 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/B, priority=13, startTime=1732160411217; duration=0sec 2024-11-21T03:40:11,311 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:11,311 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:B 2024-11-21T03:40:11,311 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:11,313 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:11,313 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/C is initiating minor compaction (all files) 2024-11-21T03:40:11,313 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/C in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:11,317 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/72281ac7c9f14f3b94dbcb63d84ba956, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/f05845015b7b42cdb10eab617a3787d1, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0d57504694d44965b3c1ba3963b477d5] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=36.0 K 2024-11-21T03:40:11,318 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 72281ac7c9f14f3b94dbcb63d84ba956, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1732160409182 2024-11-21T03:40:11,318 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting f05845015b7b42cdb10eab617a3787d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732160409902 2024-11-21T03:40:11,319 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d57504694d44965b3c1ba3963b477d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732160410277 2024-11-21T03:40:11,344 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#C#compaction#50 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:11,345 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/ecde05e792464c9eb08255ec8181310a is 50, key is test_row_0/C:col10/1732160410611/Put/seqid=0 2024-11-21T03:40:11,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741887_1063 (size=12697) 2024-11-21T03:40:11,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T03:40:11,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:11,369 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T03:40:11,375 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/ecde05e792464c9eb08255ec8181310a as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/ecde05e792464c9eb08255ec8181310a 2024-11-21T03:40:11,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:11,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:11,385 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/C of 30740eafbeb6526d1c29b0b1fec56da6 into ecde05e792464c9eb08255ec8181310a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:11,390 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:11,390 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/C, priority=13, startTime=1732160411217; duration=0sec 2024-11-21T03:40:11,390 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:11,390 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:C 2024-11-21T03:40:11,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/32e713c260184cb98932fe87ab40905f is 50, key is test_row_0/A:col10/1732160411330/Put/seqid=0 2024-11-21T03:40:11,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:11,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:11,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160471406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:11,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160471407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:11,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:11,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:11,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160471419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:11,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:11,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:11,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160471423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:11,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:11,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160471423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:11,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:11,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:11,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741888_1064 (size=12151) 2024-11-21T03:40:11,433 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/32e713c260184cb98932fe87ab40905f 2024-11-21T03:40:11,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/165709d9756845cd8758774f0f7b9c66 is 50, key is test_row_0/B:col10/1732160411330/Put/seqid=0 2024-11-21T03:40:11,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741889_1065 (size=12151) 2024-11-21T03:40:11,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:11,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160471527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:11,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:11,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160471528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:11,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:11,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:11,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160471537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:11,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:11,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160471537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:11,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:11,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160471562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:11,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:11,699 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/bcbab42508504d0a881299d502f86f5b as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/bcbab42508504d0a881299d502f86f5b 2024-11-21T03:40:11,711 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/A of 30740eafbeb6526d1c29b0b1fec56da6 into bcbab42508504d0a881299d502f86f5b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:11,712 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:11,712 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/A, priority=13, startTime=1732160411217; duration=0sec 2024-11-21T03:40:11,712 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:11,712 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:A 2024-11-21T03:40:11,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T03:40:11,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:11,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160471737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:11,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:11,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:11,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160471754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:11,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,756 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:11,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160471778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:11,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:11,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:11,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160471778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:11,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:11,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160471778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:11,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:11,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:11,910 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/165709d9756845cd8758774f0f7b9c66 2024-11-21T03:40:11,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/1fd222afa8a84b67a6aa7ede3b086545 is 50, key is test_row_0/C:col10/1732160411330/Put/seqid=0 2024-11-21T03:40:11,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741890_1066 (size=12151) 2024-11-21T03:40:12,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:12,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160472067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:12,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:12,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:12,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160472068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:12,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:12,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:12,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160472088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:12,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160472087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:12,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:12,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:12,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:12,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160472097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:12,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:12,389 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/1fd222afa8a84b67a6aa7ede3b086545 2024-11-21T03:40:12,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/32e713c260184cb98932fe87ab40905f as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/32e713c260184cb98932fe87ab40905f 2024-11-21T03:40:12,408 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/32e713c260184cb98932fe87ab40905f, entries=150, sequenceid=252, filesize=11.9 K 2024-11-21T03:40:12,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/165709d9756845cd8758774f0f7b9c66 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/165709d9756845cd8758774f0f7b9c66 2024-11-21T03:40:12,416 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/165709d9756845cd8758774f0f7b9c66, entries=150, sequenceid=252, filesize=11.9 K 2024-11-21T03:40:12,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/1fd222afa8a84b67a6aa7ede3b086545 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/1fd222afa8a84b67a6aa7ede3b086545 2024-11-21T03:40:12,427 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/1fd222afa8a84b67a6aa7ede3b086545, entries=150, sequenceid=252, filesize=11.9 K 2024-11-21T03:40:12,428 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 30740eafbeb6526d1c29b0b1fec56da6 in 1059ms, sequenceid=252, compaction requested=false 2024-11-21T03:40:12,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:12,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:12,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-21T03:40:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-21T03:40:12,432 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-21T03:40:12,433 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8370 sec 2024-11-21T03:40:12,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.8470 sec 2024-11-21T03:40:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:12,579 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:12,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T03:40:12,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/c1a3e6c3ae9a4a098a636e062c17774a is 50, key is test_row_0/A:col10/1732160411420/Put/seqid=0 2024-11-21T03:40:12,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741891_1067 (size=12301) 2024-11-21T03:40:12,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160472627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:12,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:12,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160472631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:12,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160472632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:12,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:12,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:12,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160472636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160472633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:12,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:12,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160472747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:12,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:12,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160472747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:12,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160472747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:12,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:12,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160472747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:12,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:12,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:12,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160472747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:12,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:12,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T03:40:12,757 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:12,757 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:40:12,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-21T03:40:12,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T03:40:12,762 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:40:12,764 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:40:12,764 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:40:12,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T03:40:12,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T03:40:12,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:12,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:12,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:12,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:12,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:12,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:12,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:12,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:12,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160472957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:12,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160472957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:12,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:12,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160472957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:12,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:12,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160472957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:12,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:12,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:12,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160472958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:12,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:12,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:12,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:13,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/c1a3e6c3ae9a4a098a636e062c17774a 2024-11-21T03:40:13,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/cccad8c482b144dc8951de7d6ef2014a is 50, key is test_row_0/B:col10/1732160411420/Put/seqid=0 2024-11-21T03:40:13,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741892_1068 (size=12301) 2024-11-21T03:40:13,073 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/cccad8c482b144dc8951de7d6ef2014a 2024-11-21T03:40:13,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T03:40:13,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:13,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:13,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:13,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:13,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:13,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:13,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T03:40:13,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/02650184ac804c118a34d7227dfc6028 is 50, key is test_row_0/C:col10/1732160411420/Put/seqid=0 2024-11-21T03:40:13,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741893_1069 (size=12301) 2024-11-21T03:40:13,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T03:40:13,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:13,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:13,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:13,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:13,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:13,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:13,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:13,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160473267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:13,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:13,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:13,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160473267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:13,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:13,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:13,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160473268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:13,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:13,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160473279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:13,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:13,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:13,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160473279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:13,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:13,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T03:40:13,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:13,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:13,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:13,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:13,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:13,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:13,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T03:40:13,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/02650184ac804c118a34d7227dfc6028 2024-11-21T03:40:13,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/c1a3e6c3ae9a4a098a636e062c17774a as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/c1a3e6c3ae9a4a098a636e062c17774a 2024-11-21T03:40:13,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/c1a3e6c3ae9a4a098a636e062c17774a, entries=150, sequenceid=267, filesize=12.0 K 2024-11-21T03:40:13,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/cccad8c482b144dc8951de7d6ef2014a as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/cccad8c482b144dc8951de7d6ef2014a 2024-11-21T03:40:13,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T03:40:13,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:13,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:13,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:13,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:13,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:13,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:13,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/cccad8c482b144dc8951de7d6ef2014a, entries=150, sequenceid=267, filesize=12.0 K 2024-11-21T03:40:13,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/02650184ac804c118a34d7227dfc6028 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/02650184ac804c118a34d7227dfc6028 2024-11-21T03:40:13,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/02650184ac804c118a34d7227dfc6028, entries=150, sequenceid=267, filesize=12.0 K 2024-11-21T03:40:13,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 30740eafbeb6526d1c29b0b1fec56da6 in 986ms, sequenceid=267, compaction requested=true 2024-11-21T03:40:13,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:13,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:40:13,566 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:13,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:13,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:40:13,566 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:13,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:13,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:40:13,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:13,567 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:13,567 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:13,568 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/A is initiating minor compaction (all files) 2024-11-21T03:40:13,568 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/B is initiating minor compaction (all files) 2024-11-21T03:40:13,568 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/A in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:13,568 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/B in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:13,568 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/bcbab42508504d0a881299d502f86f5b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/32e713c260184cb98932fe87ab40905f, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/c1a3e6c3ae9a4a098a636e062c17774a] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=36.3 K 2024-11-21T03:40:13,568 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/a83f2f701e174734ad546b5b3c938a5e, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/165709d9756845cd8758774f0f7b9c66, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/cccad8c482b144dc8951de7d6ef2014a] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=36.3 K 2024-11-21T03:40:13,569 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting a83f2f701e174734ad546b5b3c938a5e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732160410277 2024-11-21T03:40:13,569 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcbab42508504d0a881299d502f86f5b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732160410277 2024-11-21T03:40:13,569 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32e713c260184cb98932fe87ab40905f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732160410648 2024-11-21T03:40:13,569 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 165709d9756845cd8758774f0f7b9c66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732160410648 2024-11-21T03:40:13,570 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting cccad8c482b144dc8951de7d6ef2014a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732160411404 2024-11-21T03:40:13,570 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1a3e6c3ae9a4a098a636e062c17774a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732160411404 2024-11-21T03:40:13,586 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#B#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:13,587 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/b23c401d49c0401e8b51b57401663d37 is 50, key is test_row_0/B:col10/1732160411420/Put/seqid=0 2024-11-21T03:40:13,597 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#A#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:13,598 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/9b10cd2914bc4d928b82290dc314e5be is 50, key is test_row_0/A:col10/1732160411420/Put/seqid=0 2024-11-21T03:40:13,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741894_1070 (size=12949) 2024-11-21T03:40:13,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741895_1071 (size=12949) 2024-11-21T03:40:13,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T03:40:13,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:13,695 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T03:40:13,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/b7218ce9d4e14e7a881163b6650a5185 is 50, key is test_row_0/A:col10/1732160412631/Put/seqid=0 2024-11-21T03:40:13,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741896_1072 (size=12301) 2024-11-21T03:40:13,759 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/b7218ce9d4e14e7a881163b6650a5185 2024-11-21T03:40:13,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/52d79324325c469f866a02dddbd92fd6 is 50, key is test_row_0/B:col10/1732160412631/Put/seqid=0 2024-11-21T03:40:13,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:13,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:13,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:13,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160473816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:13,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160473817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:13,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:13,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160473825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:13,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:13,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:13,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160473838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:13,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160473825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:13,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741897_1073 (size=12301) 2024-11-21T03:40:13,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T03:40:13,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:13,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160473938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:13,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160473938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:13,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:13,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160473958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:13,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:13,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:13,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160473958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:13,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:13,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:13,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160473967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:13,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:13,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:14,048 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/b23c401d49c0401e8b51b57401663d37 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/b23c401d49c0401e8b51b57401663d37 2024-11-21T03:40:14,051 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/9b10cd2914bc4d928b82290dc314e5be as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/9b10cd2914bc4d928b82290dc314e5be 2024-11-21T03:40:14,060 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/B of 30740eafbeb6526d1c29b0b1fec56da6 into b23c401d49c0401e8b51b57401663d37(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:14,060 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:14,061 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/B, priority=13, startTime=1732160413566; duration=0sec 2024-11-21T03:40:14,061 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:14,061 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:B 2024-11-21T03:40:14,061 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:14,062 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/A of 30740eafbeb6526d1c29b0b1fec56da6 into 9b10cd2914bc4d928b82290dc314e5be(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:14,062 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:14,062 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/A, priority=13, startTime=1732160413566; duration=0sec 2024-11-21T03:40:14,063 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:14,063 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:A 2024-11-21T03:40:14,065 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:14,065 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/C is initiating minor compaction (all files) 2024-11-21T03:40:14,065 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/C in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:14,066 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/ecde05e792464c9eb08255ec8181310a, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/1fd222afa8a84b67a6aa7ede3b086545, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/02650184ac804c118a34d7227dfc6028] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=36.3 K 2024-11-21T03:40:14,066 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting ecde05e792464c9eb08255ec8181310a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1732160410277 2024-11-21T03:40:14,067 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fd222afa8a84b67a6aa7ede3b086545, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732160410648 2024-11-21T03:40:14,067 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 02650184ac804c118a34d7227dfc6028, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732160411404 2024-11-21T03:40:14,081 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#C#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:14,081 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/8a2c7aa24ba144c1bf3c14ed68e8651a is 50, key is test_row_0/C:col10/1732160411420/Put/seqid=0 2024-11-21T03:40:14,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741898_1074 (size=12949) 2024-11-21T03:40:14,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160474147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:14,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:14,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:14,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160474156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:14,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:14,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160474167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:14,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:14,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:14,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160474167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:14,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:14,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160474176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:14,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:14,264 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/52d79324325c469f866a02dddbd92fd6 2024-11-21T03:40:14,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/519d14b4b3ba4ddb9dba09a2bdcd8ff4 is 50, key is test_row_0/C:col10/1732160412631/Put/seqid=0 2024-11-21T03:40:14,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741899_1075 (size=12301) 2024-11-21T03:40:14,297 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/519d14b4b3ba4ddb9dba09a2bdcd8ff4 2024-11-21T03:40:14,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/b7218ce9d4e14e7a881163b6650a5185 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/b7218ce9d4e14e7a881163b6650a5185 2024-11-21T03:40:14,315 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/b7218ce9d4e14e7a881163b6650a5185, entries=150, sequenceid=290, filesize=12.0 K 2024-11-21T03:40:14,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/52d79324325c469f866a02dddbd92fd6 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/52d79324325c469f866a02dddbd92fd6 2024-11-21T03:40:14,324 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/52d79324325c469f866a02dddbd92fd6, entries=150, sequenceid=290, filesize=12.0 K 2024-11-21T03:40:14,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/519d14b4b3ba4ddb9dba09a2bdcd8ff4 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/519d14b4b3ba4ddb9dba09a2bdcd8ff4 2024-11-21T03:40:14,334 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/519d14b4b3ba4ddb9dba09a2bdcd8ff4, entries=150, sequenceid=290, filesize=12.0 K 2024-11-21T03:40:14,336 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=80.51 KB/82440 for 30740eafbeb6526d1c29b0b1fec56da6 in 641ms, sequenceid=290, compaction requested=false 2024-11-21T03:40:14,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:14,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:14,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-21T03:40:14,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-21T03:40:14,344 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-21T03:40:14,344 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5750 sec 2024-11-21T03:40:14,348 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.5880 sec 2024-11-21T03:40:14,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:14,460 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:14,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-21T03:40:14,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/3737ae57c77f4ddfac53de8e8aacd4e9 is 50, key is test_row_0/A:col10/1732160413817/Put/seqid=0 2024-11-21T03:40:14,508 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/8a2c7aa24ba144c1bf3c14ed68e8651a as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/8a2c7aa24ba144c1bf3c14ed68e8651a 2024-11-21T03:40:14,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:14,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160474510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:14,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:14,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160474512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:14,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:14,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:14,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:14,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160474513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:14,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:14,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:14,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160474514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:14,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:14,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:14,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160474516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:14,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:14,523 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/C of 30740eafbeb6526d1c29b0b1fec56da6 into 8a2c7aa24ba144c1bf3c14ed68e8651a(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:14,523 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:14,523 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/C, priority=13, startTime=1732160413566; duration=0sec 2024-11-21T03:40:14,523 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:14,523 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:C 2024-11-21T03:40:14,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741900_1076 (size=12301) 2024-11-21T03:40:14,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/3737ae57c77f4ddfac53de8e8aacd4e9 2024-11-21T03:40:14,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/37b68a6c2aad4f95a781f6fc8d48d8d7 is 50, key is test_row_0/B:col10/1732160413817/Put/seqid=0 2024-11-21T03:40:14,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741901_1077 (size=12301) 2024-11-21T03:40:14,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:14,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160474626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160474627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:14,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160474627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:14,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:14,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:14,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160474627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:14,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:14,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:14,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160474627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:14,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:14,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:14,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160474838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:14,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:14,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160474838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:14,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:14,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:14,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160474838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:14,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160474838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160474838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:14,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T03:40:14,928 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:14,929 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:40:14,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-21T03:40:14,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T03:40:14,933 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:40:14,937 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:40:14,937 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:40:14,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/37b68a6c2aad4f95a781f6fc8d48d8d7 2024-11-21T03:40:15,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/05f7812cfa8146228675dcf5a100c12b is 50, key is test_row_0/C:col10/1732160413817/Put/seqid=0 2024-11-21T03:40:15,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741902_1078 (size=12301) 2024-11-21T03:40:15,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T03:40:15,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T03:40:15,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:15,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:15,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:15,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:15,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:15,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160475146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:15,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:15,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160475147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:15,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:15,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:15,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160475147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:15,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:15,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:15,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:15,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:15,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160475147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:15,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160475147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:15,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:15,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T03:40:15,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T03:40:15,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:15,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:15,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:15,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:15,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:15,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:15,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T03:40:15,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:15,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:15,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:15,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:15,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:15,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:15,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/05f7812cfa8146228675dcf5a100c12b 2024-11-21T03:40:15,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/3737ae57c77f4ddfac53de8e8aacd4e9 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/3737ae57c77f4ddfac53de8e8aacd4e9 2024-11-21T03:40:15,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/3737ae57c77f4ddfac53de8e8aacd4e9, entries=150, sequenceid=308, filesize=12.0 K 2024-11-21T03:40:15,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/37b68a6c2aad4f95a781f6fc8d48d8d7 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/37b68a6c2aad4f95a781f6fc8d48d8d7 2024-11-21T03:40:15,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/37b68a6c2aad4f95a781f6fc8d48d8d7, entries=150, sequenceid=308, filesize=12.0 K 2024-11-21T03:40:15,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/05f7812cfa8146228675dcf5a100c12b as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/05f7812cfa8146228675dcf5a100c12b 2024-11-21T03:40:15,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/05f7812cfa8146228675dcf5a100c12b, entries=150, sequenceid=308, filesize=12.0 K 2024-11-21T03:40:15,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 30740eafbeb6526d1c29b0b1fec56da6 in 1000ms, sequenceid=308, compaction requested=true 2024-11-21T03:40:15,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:15,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:40:15,461 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:15,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:15,461 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:15,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:40:15,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:15,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:40:15,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:15,462 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:15,462 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:15,462 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/A is initiating minor compaction (all files) 2024-11-21T03:40:15,462 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/B is initiating minor compaction (all files) 2024-11-21T03:40:15,462 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/B in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:15,463 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/A in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:15,463 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/9b10cd2914bc4d928b82290dc314e5be, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/b7218ce9d4e14e7a881163b6650a5185, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/3737ae57c77f4ddfac53de8e8aacd4e9] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=36.7 K 2024-11-21T03:40:15,463 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/b23c401d49c0401e8b51b57401663d37, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/52d79324325c469f866a02dddbd92fd6, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/37b68a6c2aad4f95a781f6fc8d48d8d7] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=36.7 K 2024-11-21T03:40:15,463 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b10cd2914bc4d928b82290dc314e5be, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732160411404 2024-11-21T03:40:15,463 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting b23c401d49c0401e8b51b57401663d37, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732160411404 2024-11-21T03:40:15,464 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7218ce9d4e14e7a881163b6650a5185, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732160412614 2024-11-21T03:40:15,464 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 52d79324325c469f866a02dddbd92fd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732160412614 2024-11-21T03:40:15,464 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3737ae57c77f4ddfac53de8e8aacd4e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732160413817 2024-11-21T03:40:15,464 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 37b68a6c2aad4f95a781f6fc8d48d8d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732160413817 2024-11-21T03:40:15,477 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#A#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:15,477 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#B#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:15,477 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/2916228537b34a78a4df97638781dfdd is 50, key is test_row_0/A:col10/1732160413817/Put/seqid=0 2024-11-21T03:40:15,477 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/6783128a498f40cdace42808594f1737 is 50, key is test_row_0/B:col10/1732160413817/Put/seqid=0 2024-11-21T03:40:15,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741903_1079 (size=13051) 2024-11-21T03:40:15,491 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/2916228537b34a78a4df97638781dfdd as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/2916228537b34a78a4df97638781dfdd 2024-11-21T03:40:15,498 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/A of 30740eafbeb6526d1c29b0b1fec56da6 into 2916228537b34a78a4df97638781dfdd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:15,498 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:15,499 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/A, priority=13, startTime=1732160415461; duration=0sec 2024-11-21T03:40:15,499 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:15,499 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:A 2024-11-21T03:40:15,499 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:15,501 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:15,501 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/C is initiating minor compaction (all files) 2024-11-21T03:40:15,501 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/C in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:15,501 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/8a2c7aa24ba144c1bf3c14ed68e8651a, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/519d14b4b3ba4ddb9dba09a2bdcd8ff4, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/05f7812cfa8146228675dcf5a100c12b] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=36.7 K 2024-11-21T03:40:15,502 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a2c7aa24ba144c1bf3c14ed68e8651a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732160411404 2024-11-21T03:40:15,504 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 519d14b4b3ba4ddb9dba09a2bdcd8ff4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732160412614 2024-11-21T03:40:15,505 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05f7812cfa8146228675dcf5a100c12b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732160413817 2024-11-21T03:40:15,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741904_1080 (size=13051) 2024-11-21T03:40:15,528 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/6783128a498f40cdace42808594f1737 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/6783128a498f40cdace42808594f1737 2024-11-21T03:40:15,532 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#C#compaction#68 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:15,533 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/0c2c359798144025aa658127b0fd3fd0 is 50, key is test_row_0/C:col10/1732160413817/Put/seqid=0 2024-11-21T03:40:15,545 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/B of 30740eafbeb6526d1c29b0b1fec56da6 into 6783128a498f40cdace42808594f1737(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:15,545 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:15,545 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/B, priority=13, startTime=1732160415461; duration=0sec 2024-11-21T03:40:15,545 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:15,545 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:B 2024-11-21T03:40:15,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741905_1081 (size=13051) 2024-11-21T03:40:15,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T03:40:15,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T03:40:15,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:15,561 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T03:40:15,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/54f5fcf6560c496b8dd0dade23f72ba9 is 50, key is test_row_0/A:col10/1732160414494/Put/seqid=0 2024-11-21T03:40:15,570 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/0c2c359798144025aa658127b0fd3fd0 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0c2c359798144025aa658127b0fd3fd0 2024-11-21T03:40:15,582 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/C of 30740eafbeb6526d1c29b0b1fec56da6 into 0c2c359798144025aa658127b0fd3fd0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:15,582 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:15,582 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/C, priority=13, startTime=1732160415461; duration=0sec 2024-11-21T03:40:15,583 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:15,583 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:C 2024-11-21T03:40:15,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741906_1082 (size=12301) 2024-11-21T03:40:15,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:15,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:15,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:15,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160475673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:15,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160475674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:15,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:15,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:15,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160475675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:15,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:15,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160475677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:15,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:15,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:15,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160475677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:15,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:15,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:15,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160475786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:15,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:15,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:15,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160475787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:15,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:15,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160475787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:15,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160475787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:15,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:15,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160475787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:15,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:15,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160475997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:15,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:15,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:16,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:16,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160475998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:16,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:16,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:16,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160475998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:16,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:16,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160475998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:16,001 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/54f5fcf6560c496b8dd0dade23f72ba9 2024-11-21T03:40:16,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160475998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:16,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/bf7e1223167140959638620c1d5d3dd1 is 50, key is test_row_0/B:col10/1732160414494/Put/seqid=0 2024-11-21T03:40:16,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T03:40:16,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741907_1083 (size=12301) 2024-11-21T03:40:16,089 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/bf7e1223167140959638620c1d5d3dd1 2024-11-21T03:40:16,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/c1525cbf57dc44679d96f60db735bcff is 50, key is test_row_0/C:col10/1732160414494/Put/seqid=0 2024-11-21T03:40:16,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741908_1084 (size=12301) 2024-11-21T03:40:16,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:16,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160476307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:16,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:16,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160476308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:16,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:16,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160476308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:16,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:16,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160476308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:16,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:16,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:16,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160476308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:16,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:16,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:16,538 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/c1525cbf57dc44679d96f60db735bcff 2024-11-21T03:40:16,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/54f5fcf6560c496b8dd0dade23f72ba9 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/54f5fcf6560c496b8dd0dade23f72ba9 2024-11-21T03:40:16,554 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/54f5fcf6560c496b8dd0dade23f72ba9, entries=150, sequenceid=332, filesize=12.0 K 2024-11-21T03:40:16,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/bf7e1223167140959638620c1d5d3dd1 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/bf7e1223167140959638620c1d5d3dd1 2024-11-21T03:40:16,565 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/bf7e1223167140959638620c1d5d3dd1, entries=150, sequenceid=332, filesize=12.0 K 2024-11-21T03:40:16,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/c1525cbf57dc44679d96f60db735bcff as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/c1525cbf57dc44679d96f60db735bcff 2024-11-21T03:40:16,581 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/c1525cbf57dc44679d96f60db735bcff, entries=150, sequenceid=332, filesize=12.0 K 2024-11-21T03:40:16,584 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 30740eafbeb6526d1c29b0b1fec56da6 in 1023ms, sequenceid=332, compaction requested=false 2024-11-21T03:40:16,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:16,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:16,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-21T03:40:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-21T03:40:16,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-21T03:40:16,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6490 sec 2024-11-21T03:40:16,595 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.6610 sec 2024-11-21T03:40:16,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:16,818 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:16,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-21T03:40:16,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/71adc4686bc146f595879a9d2b69021f is 50, key is test_row_0/A:col10/1732160416817/Put/seqid=0 2024-11-21T03:40:16,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741909_1085 (size=12301) 2024-11-21T03:40:16,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/71adc4686bc146f595879a9d2b69021f 2024-11-21T03:40:16,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:16,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160476852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:16,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:16,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:16,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160476852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:16,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:16,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,864 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:16,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160476854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:16,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:16,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:16,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160476855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:16,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:16,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/4d201343934e42c2b1f9b8e9e2a10f59 is 50, key is test_row_0/B:col10/1732160416817/Put/seqid=0 2024-11-21T03:40:16,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160476852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:16,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:16,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741910_1086 (size=12301) 2024-11-21T03:40:16,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:16,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160476967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:16,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,971 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:16,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160476967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:16,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:16,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:16,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160476976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:16,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:16,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:16,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160476976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:16,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:16,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160476987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:16,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T03:40:17,077 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:17,077 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:40:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-21T03:40:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T03:40:17,079 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:40:17,080 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:40:17,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:40:17,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160477177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:17,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160477178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:17,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:17,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T03:40:17,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160477187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160477187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:17,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:17,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160477196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:17,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:17,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T03:40:17,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:17,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:17,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:17,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:17,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:17,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:17,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/4d201343934e42c2b1f9b8e9e2a10f59 2024-11-21T03:40:17,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/7e5e20f9e01e4aeea6647310c3008396 is 50, key is test_row_0/C:col10/1732160416817/Put/seqid=0 2024-11-21T03:40:17,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741911_1087 (size=12301) 2024-11-21T03:40:17,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/7e5e20f9e01e4aeea6647310c3008396 2024-11-21T03:40:17,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/71adc4686bc146f595879a9d2b69021f as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/71adc4686bc146f595879a9d2b69021f 2024-11-21T03:40:17,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/71adc4686bc146f595879a9d2b69021f, entries=150, sequenceid=349, filesize=12.0 K 2024-11-21T03:40:17,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/4d201343934e42c2b1f9b8e9e2a10f59 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/4d201343934e42c2b1f9b8e9e2a10f59 2024-11-21T03:40:17,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/4d201343934e42c2b1f9b8e9e2a10f59, entries=150, sequenceid=349, filesize=12.0 K 2024-11-21T03:40:17,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/7e5e20f9e01e4aeea6647310c3008396 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/7e5e20f9e01e4aeea6647310c3008396 2024-11-21T03:40:17,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/7e5e20f9e01e4aeea6647310c3008396, entries=150, sequenceid=349, filesize=12.0 K 2024-11-21T03:40:17,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 30740eafbeb6526d1c29b0b1fec56da6 in 516ms, sequenceid=349, compaction requested=true 2024-11-21T03:40:17,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:17,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:40:17,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:17,334 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:17,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:40:17,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:17,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:40:17,335 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:17,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:17,336 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:17,336 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:17,336 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/B is initiating minor compaction (all files) 2024-11-21T03:40:17,336 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/A is initiating minor compaction (all files) 2024-11-21T03:40:17,336 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/B in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:17,336 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/A in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:17,336 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/2916228537b34a78a4df97638781dfdd, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/54f5fcf6560c496b8dd0dade23f72ba9, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/71adc4686bc146f595879a9d2b69021f] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=36.8 K 2024-11-21T03:40:17,336 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/6783128a498f40cdace42808594f1737, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/bf7e1223167140959638620c1d5d3dd1, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/4d201343934e42c2b1f9b8e9e2a10f59] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=36.8 K 2024-11-21T03:40:17,337 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 6783128a498f40cdace42808594f1737, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732160413817 2024-11-21T03:40:17,337 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2916228537b34a78a4df97638781dfdd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732160413817 2024-11-21T03:40:17,337 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54f5fcf6560c496b8dd0dade23f72ba9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732160414494 2024-11-21T03:40:17,337 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting bf7e1223167140959638620c1d5d3dd1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732160414494 2024-11-21T03:40:17,338 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d201343934e42c2b1f9b8e9e2a10f59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732160415673 2024-11-21T03:40:17,338 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71adc4686bc146f595879a9d2b69021f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732160415673 2024-11-21T03:40:17,346 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#B#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:17,347 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/7a143d14bc484f10a38590a53897ac62 is 50, key is test_row_0/B:col10/1732160416817/Put/seqid=0 2024-11-21T03:40:17,347 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#A#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:17,348 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/f65de732c5bc447b8138786d5138ace4 is 50, key is test_row_0/A:col10/1732160416817/Put/seqid=0 2024-11-21T03:40:17,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741912_1088 (size=13153) 2024-11-21T03:40:17,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741913_1089 (size=13153) 2024-11-21T03:40:17,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T03:40:17,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:17,386 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-21T03:40:17,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/74b60654f895436aa97c0a2aeb7dfc6f is 50, key is test_row_0/A:col10/1732160416852/Put/seqid=0 2024-11-21T03:40:17,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741914_1090 (size=12301) 2024-11-21T03:40:17,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T03:40:17,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:17,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:17,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:17,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160477507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:17,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:17,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:17,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160477510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:17,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:17,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:17,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160477512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:17,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:17,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:17,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160477512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:17,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:17,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160477513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:17,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:17,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160477617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:17,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:17,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160477627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:17,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:17,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:17,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160477627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:17,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:17,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:17,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160477627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:17,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:17,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T03:40:17,762 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/7a143d14bc484f10a38590a53897ac62 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/7a143d14bc484f10a38590a53897ac62 2024-11-21T03:40:17,768 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/f65de732c5bc447b8138786d5138ace4 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/f65de732c5bc447b8138786d5138ace4 2024-11-21T03:40:17,773 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/B of 30740eafbeb6526d1c29b0b1fec56da6 into 7a143d14bc484f10a38590a53897ac62(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:17,773 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:17,773 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/B, priority=13, startTime=1732160417334; duration=0sec 2024-11-21T03:40:17,774 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:17,774 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:B 2024-11-21T03:40:17,774 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:17,777 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:17,777 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/C is initiating minor compaction (all files) 2024-11-21T03:40:17,778 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/C in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:17,778 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0c2c359798144025aa658127b0fd3fd0, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/c1525cbf57dc44679d96f60db735bcff, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/7e5e20f9e01e4aeea6647310c3008396] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=36.8 K 2024-11-21T03:40:17,779 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/A of 30740eafbeb6526d1c29b0b1fec56da6 into f65de732c5bc447b8138786d5138ace4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:17,780 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c2c359798144025aa658127b0fd3fd0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732160413817 2024-11-21T03:40:17,780 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting c1525cbf57dc44679d96f60db735bcff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732160414494 2024-11-21T03:40:17,779 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:17,781 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e5e20f9e01e4aeea6647310c3008396, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732160415673 2024-11-21T03:40:17,781 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/A, priority=13, startTime=1732160417334; duration=0sec 2024-11-21T03:40:17,781 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:17,781 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:A 2024-11-21T03:40:17,791 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#C#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:17,792 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/20652d2532e448b282d28e98ca82af4e is 50, key is test_row_0/C:col10/1732160416817/Put/seqid=0 2024-11-21T03:40:17,797 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/74b60654f895436aa97c0a2aeb7dfc6f 2024-11-21T03:40:17,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/9c8960a12bfe4616babdca6ca8e9481c is 50, key is test_row_0/B:col10/1732160416852/Put/seqid=0 2024-11-21T03:40:17,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741915_1091 (size=13153) 2024-11-21T03:40:17,824 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/20652d2532e448b282d28e98ca82af4e as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/20652d2532e448b282d28e98ca82af4e 2024-11-21T03:40:17,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741916_1092 (size=12301) 2024-11-21T03:40:17,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160477827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:17,830 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/9c8960a12bfe4616babdca6ca8e9481c 2024-11-21T03:40:17,833 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/C of 30740eafbeb6526d1c29b0b1fec56da6 into 20652d2532e448b282d28e98ca82af4e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:17,833 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:17,833 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/C, priority=13, startTime=1732160417335; duration=0sec 2024-11-21T03:40:17,833 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:17,833 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:C 2024-11-21T03:40:17,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:17,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160477837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:17,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:17,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:17,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160477837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:17,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:17,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:17,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160477837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:17,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:17,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:17,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/c1206659a26749c4938395f436953364 is 50, key is test_row_0/C:col10/1732160416852/Put/seqid=0 2024-11-21T03:40:17,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741917_1093 (size=12301) 2024-11-21T03:40:18,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:18,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160478027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:18,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:18,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:18,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:18,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:18,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160478137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:18,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:18,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:18,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:18,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160478147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:18,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160478148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:18,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:18,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:18,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:18,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160478148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:18,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:18,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:18,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:18,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T03:40:18,260 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/c1206659a26749c4938395f436953364 2024-11-21T03:40:18,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/74b60654f895436aa97c0a2aeb7dfc6f as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/74b60654f895436aa97c0a2aeb7dfc6f 2024-11-21T03:40:18,276 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/74b60654f895436aa97c0a2aeb7dfc6f, entries=150, sequenceid=369, filesize=12.0 K 2024-11-21T03:40:18,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/9c8960a12bfe4616babdca6ca8e9481c as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/9c8960a12bfe4616babdca6ca8e9481c 2024-11-21T03:40:18,289 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/9c8960a12bfe4616babdca6ca8e9481c, entries=150, sequenceid=369, filesize=12.0 K 2024-11-21T03:40:18,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/c1206659a26749c4938395f436953364 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/c1206659a26749c4938395f436953364 2024-11-21T03:40:18,303 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/c1206659a26749c4938395f436953364, entries=150, sequenceid=369, filesize=12.0 K 2024-11-21T03:40:18,304 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 30740eafbeb6526d1c29b0b1fec56da6 in 918ms, sequenceid=369, compaction requested=false 2024-11-21T03:40:18,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:18,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:18,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-21T03:40:18,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-21T03:40:18,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-21T03:40:18,308 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2260 sec 2024-11-21T03:40:18,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.2310 sec 2024-11-21T03:40:18,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:18,661 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:18,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-21T03:40:18,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/c1864d5954a044ea8969cb2c7160d829 is 50, key is test_row_0/A:col10/1732160418660/Put/seqid=0 2024-11-21T03:40:18,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741918_1094 (size=12301) 2024-11-21T03:40:18,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/c1864d5954a044ea8969cb2c7160d829 2024-11-21T03:40:18,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/49c958c785964d81b6188b131e1dd62b is 50, key is test_row_0/B:col10/1732160418660/Put/seqid=0 2024-11-21T03:40:18,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741919_1095 (size=12301) 2024-11-21T03:40:18,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/49c958c785964d81b6188b131e1dd62b 2024-11-21T03:40:18,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/7350291e272c42d0b640cf2735932342 is 50, key is test_row_0/C:col10/1732160418660/Put/seqid=0 2024-11-21T03:40:18,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741920_1096 (size=12301) 2024-11-21T03:40:18,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160478747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:18,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:18,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:18,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:18,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160478751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:18,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:18,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:18,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:18,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160478754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:18,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160478755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:18,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:18,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:18,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:18,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:18,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:18,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:18,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160478857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:18,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:18,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:18,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:18,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160478867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:18,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160478877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:18,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:18,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:18,883 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:18,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160478877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:18,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:18,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:18,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:19,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160479047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:19,055 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_0' in 30740eafbeb6526d1c29b0b1fec56da6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:40:19,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,057 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:19,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160479067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:19,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:19,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160479077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:19,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:19,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160479087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:19,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160479087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:19,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/7350291e272c42d0b640cf2735932342 2024-11-21T03:40:19,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/c1864d5954a044ea8969cb2c7160d829 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/c1864d5954a044ea8969cb2c7160d829 2024-11-21T03:40:19,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/c1864d5954a044ea8969cb2c7160d829, entries=150, sequenceid=391, filesize=12.0 K 2024-11-21T03:40:19,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/49c958c785964d81b6188b131e1dd62b as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/49c958c785964d81b6188b131e1dd62b 2024-11-21T03:40:19,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/49c958c785964d81b6188b131e1dd62b, entries=150, sequenceid=391, filesize=12.0 K 2024-11-21T03:40:19,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/7350291e272c42d0b640cf2735932342 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/7350291e272c42d0b640cf2735932342 2024-11-21T03:40:19,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/7350291e272c42d0b640cf2735932342, entries=150, sequenceid=391, filesize=12.0 K 2024-11-21T03:40:19,154 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 30740eafbeb6526d1c29b0b1fec56da6 in 493ms, sequenceid=391, compaction requested=true 2024-11-21T03:40:19,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:19,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:40:19,154 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:19,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:19,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:40:19,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:19,155 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:19,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:40:19,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:19,156 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:19,156 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/A is initiating minor compaction (all files) 2024-11-21T03:40:19,156 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/A in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:19,156 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/f65de732c5bc447b8138786d5138ace4, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/74b60654f895436aa97c0a2aeb7dfc6f, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/c1864d5954a044ea8969cb2c7160d829] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=36.9 K 2024-11-21T03:40:19,157 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:19,157 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/B is initiating minor compaction (all files) 2024-11-21T03:40:19,157 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/B in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:19,158 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/7a143d14bc484f10a38590a53897ac62, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/9c8960a12bfe4616babdca6ca8e9481c, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/49c958c785964d81b6188b131e1dd62b] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=36.9 K 2024-11-21T03:40:19,158 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting f65de732c5bc447b8138786d5138ace4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732160415673 2024-11-21T03:40:19,159 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a143d14bc484f10a38590a53897ac62, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732160415673 2024-11-21T03:40:19,159 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74b60654f895436aa97c0a2aeb7dfc6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732160416842 2024-11-21T03:40:19,160 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c8960a12bfe4616babdca6ca8e9481c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732160416842 2024-11-21T03:40:19,160 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1864d5954a044ea8969cb2c7160d829, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732160417499 2024-11-21T03:40:19,160 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 49c958c785964d81b6188b131e1dd62b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732160417499 2024-11-21T03:40:19,180 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#A#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:19,181 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/107da299bafd482bb2af07da902eb880 is 50, key is test_row_0/A:col10/1732160418660/Put/seqid=0 2024-11-21T03:40:19,196 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#B#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:19,197 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/0584f330b88b4db2b331f794614c91d6 is 50, key is test_row_0/B:col10/1732160418660/Put/seqid=0 2024-11-21T03:40:19,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741921_1097 (size=13255) 2024-11-21T03:40:19,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T03:40:19,227 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:19,228 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:40:19,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-21T03:40:19,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T03:40:19,232 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:40:19,233 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:40:19,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:40:19,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741922_1098 (size=13255) 2024-11-21T03:40:19,256 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/0584f330b88b4db2b331f794614c91d6 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/0584f330b88b4db2b331f794614c91d6 2024-11-21T03:40:19,266 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/B of 30740eafbeb6526d1c29b0b1fec56da6 into 0584f330b88b4db2b331f794614c91d6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:19,266 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:19,266 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/B, priority=13, startTime=1732160419154; duration=0sec 2024-11-21T03:40:19,266 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:19,266 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:B 2024-11-21T03:40:19,266 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:19,271 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:19,271 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/C is initiating minor compaction (all files) 2024-11-21T03:40:19,271 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/C in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:19,271 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/20652d2532e448b282d28e98ca82af4e, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/c1206659a26749c4938395f436953364, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/7350291e272c42d0b640cf2735932342] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=36.9 K 2024-11-21T03:40:19,278 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 20652d2532e448b282d28e98ca82af4e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732160415673 2024-11-21T03:40:19,278 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting c1206659a26749c4938395f436953364, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732160416842 2024-11-21T03:40:19,279 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 7350291e272c42d0b640cf2735932342, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732160417499 2024-11-21T03:40:19,298 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#C#compaction#86 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:19,299 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/0ba37b2805a54de2bd1728fb60763874 is 50, key is test_row_0/C:col10/1732160418660/Put/seqid=0 2024-11-21T03:40:19,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741923_1099 (size=13255) 2024-11-21T03:40:19,320 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/0ba37b2805a54de2bd1728fb60763874 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0ba37b2805a54de2bd1728fb60763874 2024-11-21T03:40:19,336 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/C of 30740eafbeb6526d1c29b0b1fec56da6 into 0ba37b2805a54de2bd1728fb60763874(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:19,336 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:19,336 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/C, priority=13, startTime=1732160419155; duration=0sec 2024-11-21T03:40:19,336 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:19,336 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:C 2024-11-21T03:40:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T03:40:19,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:19,379 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:19,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-21T03:40:19,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/8a6b1732378b4381861cc89279102cf3 is 50, key is test_row_0/A:col10/1732160418746/Put/seqid=0 2024-11-21T03:40:19,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T03:40:19,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:19,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:19,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:19,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:19,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:19,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:19,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741924_1100 (size=12301) 2024-11-21T03:40:19,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/8a6b1732378b4381861cc89279102cf3 2024-11-21T03:40:19,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:19,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160479416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:19,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:19,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/499ec27657e54e4b90ba430b79e08b4f is 50, key is test_row_0/B:col10/1732160418746/Put/seqid=0 2024-11-21T03:40:19,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160479417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:19,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:19,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160479417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:19,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:19,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160479417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:19,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:19,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741925_1101 (size=12301) 2024-11-21T03:40:19,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:19,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160479526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:19,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:19,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:19,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160479526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:19,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:19,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:19,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160479526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:19,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:19,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:19,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160479527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:19,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,530 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:19,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T03:40:19,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:19,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:19,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:19,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:19,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:19,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:19,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T03:40:19,636 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/107da299bafd482bb2af07da902eb880 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/107da299bafd482bb2af07da902eb880 2024-11-21T03:40:19,644 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/A of 30740eafbeb6526d1c29b0b1fec56da6 into 107da299bafd482bb2af07da902eb880(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:19,645 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:19,645 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/A, priority=13, startTime=1732160419154; duration=0sec 2024-11-21T03:40:19,645 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:19,645 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:A 2024-11-21T03:40:19,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T03:40:19,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:19,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:19,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:19,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:19,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:19,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:19,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160479736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:19,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160479737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:19,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160479737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:19,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160479737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:19,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/499ec27657e54e4b90ba430b79e08b4f 2024-11-21T03:40:19,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/b71c77d1ba8e4bf08c2998c398d40ca2 is 50, key is test_row_0/C:col10/1732160418746/Put/seqid=0 2024-11-21T03:40:19,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T03:40:19,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:19,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:19,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:19,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:19,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:19,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:19,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741926_1102 (size=12301) 2024-11-21T03:40:19,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/b71c77d1ba8e4bf08c2998c398d40ca2 2024-11-21T03:40:19,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T03:40:19,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/8a6b1732378b4381861cc89279102cf3 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/8a6b1732378b4381861cc89279102cf3 2024-11-21T03:40:19,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/8a6b1732378b4381861cc89279102cf3, entries=150, sequenceid=412, filesize=12.0 K 2024-11-21T03:40:19,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/499ec27657e54e4b90ba430b79e08b4f as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/499ec27657e54e4b90ba430b79e08b4f 2024-11-21T03:40:19,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/499ec27657e54e4b90ba430b79e08b4f, entries=150, sequenceid=412, filesize=12.0 K 2024-11-21T03:40:19,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/b71c77d1ba8e4bf08c2998c398d40ca2 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/b71c77d1ba8e4bf08c2998c398d40ca2 2024-11-21T03:40:19,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/b71c77d1ba8e4bf08c2998c398d40ca2, entries=150, sequenceid=412, filesize=12.0 K 2024-11-21T03:40:19,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 30740eafbeb6526d1c29b0b1fec56da6 in 510ms, sequenceid=412, compaction requested=false 2024-11-21T03:40:19,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:20,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T03:40:20,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:20,004 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-21T03:40:20,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/315f3b87e0d849c5be9a13aad60f386b is 50, key is test_row_0/A:col10/1732160419415/Put/seqid=0 2024-11-21T03:40:20,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741927_1103 (size=12301) 2024-11-21T03:40:20,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:20,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:20,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:20,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:20,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160480072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:20,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160480073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:20,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,082 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:20,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:20,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160480082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:20,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:20,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160480084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:20,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:20,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:20,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160480186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:20,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:20,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:20,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:20,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160480197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:20,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160480197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:20,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:20,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:20,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:20,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160480197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:20,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T03:40:20,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160480397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:20,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160480407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:20,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160480407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:20,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160480407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:20,415 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/315f3b87e0d849c5be9a13aad60f386b 2024-11-21T03:40:20,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/b0845b6f1a104ac19c36c5822527981d is 50, key is test_row_0/B:col10/1732160419415/Put/seqid=0 2024-11-21T03:40:20,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741928_1104 (size=12301) 2024-11-21T03:40:20,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:20,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160480707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:20,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:20,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:20,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160480717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:20,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160480717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:20,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160480717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:20,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:20,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:20,830 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/b0845b6f1a104ac19c36c5822527981d 2024-11-21T03:40:20,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/5e461e6369cd4acfaf5ad659ea77f896 is 50, key is test_row_0/C:col10/1732160419415/Put/seqid=0 2024-11-21T03:40:20,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741929_1105 (size=12301) 2024-11-21T03:40:21,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:21,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60798 deadline: 1732160481067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:21,069 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_0' in 30740eafbeb6526d1c29b0b1fec56da6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:40:21,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:21,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:21,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:21,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:21,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160481217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:21,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:21,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:21,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:21,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:21,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160481227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:21,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:21,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160481227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:21,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:21,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160481227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:21,249 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/5e461e6369cd4acfaf5ad659ea77f896 2024-11-21T03:40:21,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/315f3b87e0d849c5be9a13aad60f386b as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/315f3b87e0d849c5be9a13aad60f386b 2024-11-21T03:40:21,265 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/315f3b87e0d849c5be9a13aad60f386b, entries=150, sequenceid=431, filesize=12.0 K 2024-11-21T03:40:21,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/b0845b6f1a104ac19c36c5822527981d as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/b0845b6f1a104ac19c36c5822527981d 2024-11-21T03:40:21,273 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/b0845b6f1a104ac19c36c5822527981d, entries=150, sequenceid=431, filesize=12.0 K 2024-11-21T03:40:21,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/5e461e6369cd4acfaf5ad659ea77f896 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5e461e6369cd4acfaf5ad659ea77f896 2024-11-21T03:40:21,280 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5e461e6369cd4acfaf5ad659ea77f896, entries=150, sequenceid=431, filesize=12.0 K 2024-11-21T03:40:21,282 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 30740eafbeb6526d1c29b0b1fec56da6 in 1278ms, sequenceid=431, compaction requested=true 2024-11-21T03:40:21,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:21,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:21,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-21T03:40:21,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-21T03:40:21,286 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-21T03:40:21,286 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0510 sec 2024-11-21T03:40:21,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.0590 sec 2024-11-21T03:40:21,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T03:40:21,377 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:21,377 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:40:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-21T03:40:21,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T03:40:21,379 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:40:21,380 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:40:21,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:40:21,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T03:40:21,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-21T03:40:21,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:21,532 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-21T03:40:21,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/7e597a0377234bc8bf93b6ad4048367d is 50, key is test_row_0/A:col10/1732160420080/Put/seqid=0 2024-11-21T03:40:21,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741930_1106 (size=12301) 2024-11-21T03:40:21,549 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/7e597a0377234bc8bf93b6ad4048367d 2024-11-21T03:40:21,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/7f62a254b6c242a1aa18fe6009a9102f is 50, key is test_row_0/B:col10/1732160420080/Put/seqid=0 2024-11-21T03:40:21,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741931_1107 (size=12301) 2024-11-21T03:40:21,585 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/7f62a254b6c242a1aa18fe6009a9102f 2024-11-21T03:40:21,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/852bd229b95d4ea4a2d516c6426615e3 is 50, key is test_row_0/C:col10/1732160420080/Put/seqid=0 2024-11-21T03:40:21,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741932_1108 (size=12301) 2024-11-21T03:40:21,623 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/852bd229b95d4ea4a2d516c6426615e3 2024-11-21T03:40:21,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/7e597a0377234bc8bf93b6ad4048367d as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/7e597a0377234bc8bf93b6ad4048367d 2024-11-21T03:40:21,642 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/7e597a0377234bc8bf93b6ad4048367d, entries=150, sequenceid=450, filesize=12.0 K 2024-11-21T03:40:21,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/7f62a254b6c242a1aa18fe6009a9102f as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/7f62a254b6c242a1aa18fe6009a9102f 2024-11-21T03:40:21,650 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/7f62a254b6c242a1aa18fe6009a9102f, entries=150, sequenceid=450, filesize=12.0 K 2024-11-21T03:40:21,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/852bd229b95d4ea4a2d516c6426615e3 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/852bd229b95d4ea4a2d516c6426615e3 2024-11-21T03:40:21,668 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/852bd229b95d4ea4a2d516c6426615e3, entries=150, sequenceid=450, filesize=12.0 K 2024-11-21T03:40:21,670 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=0 B/0 for 30740eafbeb6526d1c29b0b1fec56da6 in 138ms, sequenceid=450, compaction requested=true 2024-11-21T03:40:21,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:21,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:21,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-21T03:40:21,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-21T03:40:21,677 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-21T03:40:21,677 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 293 msec 2024-11-21T03:40:21,679 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 301 msec 2024-11-21T03:40:21,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T03:40:21,697 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:21,697 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:40:21,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-21T03:40:21,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-21T03:40:21,699 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:40:21,700 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:40:21,701 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:40:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-21T03:40:21,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-21T03:40:21,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:21,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:21,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:21,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-21T03:40:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-21T03:40:21,859 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-21T03:40:21,859 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-21T03:40:21,867 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 167 msec 2024-11-21T03:40:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-21T03:40:22,017 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:22,018 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:40:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees 2024-11-21T03:40:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-21T03:40:22,020 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:40:22,022 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:40:22,022 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:40:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-21T03:40:22,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-21T03:40:22,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:22,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:22,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:22,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-11-21T03:40:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-11-21T03:40:22,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-21T03:40:22,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-21T03:40:22,181 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees in 162 msec 2024-11-21T03:40:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:22,240 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:22,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:40:22,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/a4cf434374d647b58abcdbb9dc110fee is 50, key is test_row_0/A:col10/1732160422239/Put/seqid=0 2024-11-21T03:40:22,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741933_1109 (size=14741) 2024-11-21T03:40:22,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160482307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:22,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160482309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:22,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160482311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:22,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160482311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-21T03:40:22,338 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:22,338 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:40:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=31, table=TestAcidGuarantees 2024-11-21T03:40:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-21T03:40:22,343 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=31, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=31, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:40:22,344 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=31, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=31, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:40:22,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:40:22,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160482427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:22,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160482427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:22,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:22,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:22,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160482437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:22,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160482437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:22,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:22,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-21T03:40:22,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-21T03:40:22,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:22,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:22,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:22,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:22,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:22,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:22,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160482636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:22,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160482647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:22,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160482647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:22,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160482647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:22,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-21T03:40:22,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:22,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:22,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:22,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:22,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-21T03:40:22,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/a4cf434374d647b58abcdbb9dc110fee 2024-11-21T03:40:22,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/ac3e9679fa4a4f98abdb636c0d5094d3 is 50, key is test_row_0/B:col10/1732160422239/Put/seqid=0 2024-11-21T03:40:22,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741934_1110 (size=12301) 2024-11-21T03:40:22,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-21T03:40:22,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:22,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:22,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:22,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:22,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:22,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:22,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60826 deadline: 1732160482947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:22,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:22,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-21T03:40:22,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:22,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:22,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:22,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:22,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:22,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60852 deadline: 1732160482957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:22,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60832 deadline: 1732160482957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:22,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60810 deadline: 1732160482957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:22,960 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-21T03:40:23,079 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/ac3e9679fa4a4f98abdb636c0d5094d3 2024-11-21T03:40:23,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/e3dbbb1e8f164c1e8e4746b4d487f502 is 50, key is test_row_0/C:col10/1732160422239/Put/seqid=0 2024-11-21T03:40:23,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741935_1111 (size=12301) 2024-11-21T03:40:23,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/e3dbbb1e8f164c1e8e4746b4d487f502 2024-11-21T03:40:23,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/a4cf434374d647b58abcdbb9dc110fee as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/a4cf434374d647b58abcdbb9dc110fee 2024-11-21T03:40:23,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/a4cf434374d647b58abcdbb9dc110fee, entries=200, sequenceid=463, filesize=14.4 K 2024-11-21T03:40:23,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/ac3e9679fa4a4f98abdb636c0d5094d3 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/ac3e9679fa4a4f98abdb636c0d5094d3 2024-11-21T03:40:23,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/ac3e9679fa4a4f98abdb636c0d5094d3, entries=150, sequenceid=463, filesize=12.0 K 2024-11-21T03:40:23,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/e3dbbb1e8f164c1e8e4746b4d487f502 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/e3dbbb1e8f164c1e8e4746b4d487f502 2024-11-21T03:40:23,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-21T03:40:23,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:23,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:23,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:23,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:23,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:23,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:23,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/e3dbbb1e8f164c1e8e4746b4d487f502, entries=150, sequenceid=463, filesize=12.0 K 2024-11-21T03:40:23,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 30740eafbeb6526d1c29b0b1fec56da6 in 877ms, sequenceid=463, compaction requested=true 2024-11-21T03:40:23,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:23,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:40:23,118 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T03:40:23,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:23,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:40:23,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:23,118 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T03:40:23,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:40:23,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:23,120 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 64899 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T03:40:23,120 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/A is initiating minor compaction (all files) 2024-11-21T03:40:23,120 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/A in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:23,120 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/107da299bafd482bb2af07da902eb880, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/8a6b1732378b4381861cc89279102cf3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/315f3b87e0d849c5be9a13aad60f386b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/7e597a0377234bc8bf93b6ad4048367d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/a4cf434374d647b58abcdbb9dc110fee] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=63.4 K 2024-11-21T03:40:23,121 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62459 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T03:40:23,121 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/B is initiating minor compaction (all files) 2024-11-21T03:40:23,121 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/B in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:23,121 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 107da299bafd482bb2af07da902eb880, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732160417499 2024-11-21T03:40:23,121 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/0584f330b88b4db2b331f794614c91d6, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/499ec27657e54e4b90ba430b79e08b4f, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/b0845b6f1a104ac19c36c5822527981d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/7f62a254b6c242a1aa18fe6009a9102f, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/ac3e9679fa4a4f98abdb636c0d5094d3] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=61.0 K 2024-11-21T03:40:23,121 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a6b1732378b4381861cc89279102cf3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732160418686 2024-11-21T03:40:23,121 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0584f330b88b4db2b331f794614c91d6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732160417499 2024-11-21T03:40:23,122 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 315f3b87e0d849c5be9a13aad60f386b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732160419411 2024-11-21T03:40:23,122 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 499ec27657e54e4b90ba430b79e08b4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732160418686 2024-11-21T03:40:23,122 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e597a0377234bc8bf93b6ad4048367d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1732160420051 2024-11-21T03:40:23,122 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4cf434374d647b58abcdbb9dc110fee, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732160422227 2024-11-21T03:40:23,122 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting b0845b6f1a104ac19c36c5822527981d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732160419411 2024-11-21T03:40:23,123 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f62a254b6c242a1aa18fe6009a9102f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1732160420051 2024-11-21T03:40:23,123 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting ac3e9679fa4a4f98abdb636c0d5094d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732160422227 2024-11-21T03:40:23,142 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#A#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:23,142 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/8f7523b34fdd4a099d9406745e15d286 is 50, key is test_row_0/A:col10/1732160422239/Put/seqid=0 2024-11-21T03:40:23,142 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#B#compaction#100 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:23,143 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/364faa70f8a34e4b9fafed70288da345 is 50, key is test_row_0/B:col10/1732160422239/Put/seqid=0 2024-11-21T03:40:23,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741936_1112 (size=13425) 2024-11-21T03:40:23,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741937_1113 (size=13425) 2024-11-21T03:40:23,154 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/8f7523b34fdd4a099d9406745e15d286 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/8f7523b34fdd4a099d9406745e15d286 2024-11-21T03:40:23,159 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/A of 30740eafbeb6526d1c29b0b1fec56da6 into 8f7523b34fdd4a099d9406745e15d286(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:23,159 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:23,159 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/A, priority=11, startTime=1732160423118; duration=0sec 2024-11-21T03:40:23,160 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:23,160 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:A 2024-11-21T03:40:23,160 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T03:40:23,162 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62459 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T03:40:23,162 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/C is initiating minor compaction (all files) 2024-11-21T03:40:23,162 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/C in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:23,163 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0ba37b2805a54de2bd1728fb60763874, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/b71c77d1ba8e4bf08c2998c398d40ca2, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5e461e6369cd4acfaf5ad659ea77f896, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/852bd229b95d4ea4a2d516c6426615e3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/e3dbbb1e8f164c1e8e4746b4d487f502] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=61.0 K 2024-11-21T03:40:23,163 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ba37b2805a54de2bd1728fb60763874, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732160417499 2024-11-21T03:40:23,163 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting b71c77d1ba8e4bf08c2998c398d40ca2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1732160418686 2024-11-21T03:40:23,164 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e461e6369cd4acfaf5ad659ea77f896, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732160419411 2024-11-21T03:40:23,164 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 852bd229b95d4ea4a2d516c6426615e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1732160420051 2024-11-21T03:40:23,164 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3dbbb1e8f164c1e8e4746b4d487f502, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732160422227 2024-11-21T03:40:23,173 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#C#compaction#101 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:23,174 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/8dc74a722dd44836b72843166e4e7028 is 50, key is test_row_0/C:col10/1732160422239/Put/seqid=0 2024-11-21T03:40:23,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741938_1114 (size=13425) 2024-11-21T03:40:23,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-21T03:40:23,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:23,264 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T03:40:23,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/3c589201e9334e3caabcb4876030b219 is 50, key is test_row_0/A:col10/1732160422297/Put/seqid=0 2024-11-21T03:40:23,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741939_1115 (size=12301) 2024-11-21T03:40:23,274 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/3c589201e9334e3caabcb4876030b219 2024-11-21T03:40:23,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/d17bfc64e3444d94bf8a930c5deb0b04 is 50, key is test_row_0/B:col10/1732160422297/Put/seqid=0 2024-11-21T03:40:23,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741940_1116 (size=12301) 2024-11-21T03:40:23,295 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/d17bfc64e3444d94bf8a930c5deb0b04 2024-11-21T03:40:23,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/0d2043db3546436a9ec0a71e4fe15cfa is 50, key is test_row_0/C:col10/1732160422297/Put/seqid=0 2024-11-21T03:40:23,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741941_1117 (size=12301) 2024-11-21T03:40:23,326 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/0d2043db3546436a9ec0a71e4fe15cfa 2024-11-21T03:40:23,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/3c589201e9334e3caabcb4876030b219 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/3c589201e9334e3caabcb4876030b219 2024-11-21T03:40:23,353 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/3c589201e9334e3caabcb4876030b219, entries=150, sequenceid=490, filesize=12.0 K 2024-11-21T03:40:23,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/d17bfc64e3444d94bf8a930c5deb0b04 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/d17bfc64e3444d94bf8a930c5deb0b04 2024-11-21T03:40:23,363 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/d17bfc64e3444d94bf8a930c5deb0b04, entries=150, sequenceid=490, filesize=12.0 K 2024-11-21T03:40:23,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/0d2043db3546436a9ec0a71e4fe15cfa as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0d2043db3546436a9ec0a71e4fe15cfa 2024-11-21T03:40:23,374 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0d2043db3546436a9ec0a71e4fe15cfa, entries=150, sequenceid=490, filesize=12.0 K 2024-11-21T03:40:23,376 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=0 B/0 for 30740eafbeb6526d1c29b0b1fec56da6 in 111ms, sequenceid=490, compaction requested=false 2024-11-21T03:40:23,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:23,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:23,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=32 2024-11-21T03:40:23,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=32 2024-11-21T03:40:23,381 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-21T03:40:23,381 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0340 sec 2024-11-21T03:40:23,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=31, table=TestAcidGuarantees in 1.0430 sec 2024-11-21T03:40:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-21T03:40:23,477 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:23,477 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:40:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=33, table=TestAcidGuarantees 2024-11-21T03:40:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-21T03:40:23,479 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:23,480 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=33, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=33, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:40:23,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:40:23,480 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=33, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=33, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:40:23,480 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:40:23,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/e77cc8c7b4824fc78a5f8c0057623827 is 50, key is test_row_0/A:col10/1732160423479/Put/seqid=0 2024-11-21T03:40:23,495 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-21T03:40:23,495 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:40:23,496 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-21T03:40:23,496 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:40:23,499 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-21T03:40:23,499 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:40:23,501 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-21T03:40:23,501 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-21T03:40:23,501 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:40:23,501 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:40:23,501 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:23,502 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:23,502 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:40:23,502 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:23,502 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:23,502 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:40:23,503 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:23,503 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:23,503 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:23,503 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:23,503 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:40:23,503 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:40:23,503 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:23,504 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:23,504 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-21T03:40:23,505 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:40:23,505 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:23,505 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:40:23,505 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-21T03:40:23,505 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:23,505 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:40:23,505 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:40:23,505 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:23,506 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:23,506 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:40:23,507 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-21T03:40:23,507 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:40:23,507 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:23,507 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:23,507 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:40:23,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741942_1118 (size=14737) 2024-11-21T03:40:23,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/e77cc8c7b4824fc78a5f8c0057623827 2024-11-21T03:40:23,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/36e3ca87482a4ff1b057797d1af0741d is 50, key is test_row_0/B:col10/1732160423479/Put/seqid=0 2024-11-21T03:40:23,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741943_1119 (size=9857) 2024-11-21T03:40:23,554 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/364faa70f8a34e4b9fafed70288da345 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/364faa70f8a34e4b9fafed70288da345 2024-11-21T03:40:23,560 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/B of 30740eafbeb6526d1c29b0b1fec56da6 into 364faa70f8a34e4b9fafed70288da345(size=13.1 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:23,560 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:23,560 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/B, priority=11, startTime=1732160423118; duration=0sec 2024-11-21T03:40:23,561 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:23,561 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:B 2024-11-21T03:40:23,586 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/8dc74a722dd44836b72843166e4e7028 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/8dc74a722dd44836b72843166e4e7028 2024-11-21T03:40:23,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-21T03:40:23,594 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/C of 30740eafbeb6526d1c29b0b1fec56da6 into 8dc74a722dd44836b72843166e4e7028(size=13.1 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:23,594 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:23,594 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/C, priority=11, startTime=1732160423118; duration=0sec 2024-11-21T03:40:23,594 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:23,594 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:C 2024-11-21T03:40:23,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=34 2024-11-21T03:40:23,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:23,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:23,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:23,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=34}] handler.RSProcedureHandler(58): pid=34 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:23,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=34 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:23,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=34 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:23,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=34 2024-11-21T03:40:23,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:23,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:23,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:23,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=34}] handler.RSProcedureHandler(58): pid=34 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:23,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=34 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=34 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-21T03:40:23,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/36e3ca87482a4ff1b057797d1af0741d 2024-11-21T03:40:23,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/b47988fdf2664909b6524174dcd1de11 is 50, key is test_row_0/C:col10/1732160423479/Put/seqid=0 2024-11-21T03:40:23,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741944_1120 (size=9857) 2024-11-21T03:40:23,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=34 2024-11-21T03:40:23,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:23,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:23,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:23,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=34}] handler.RSProcedureHandler(58): pid=34 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:23,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=34 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:23,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=34 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:24,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=34 2024-11-21T03:40:24,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:24,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:24,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:24,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=34}] handler.RSProcedureHandler(58): pid=34 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:24,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=34 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:24,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=34 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:24,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-21T03:40:24,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=34 2024-11-21T03:40:24,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:24,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:24,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:24,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=34}] handler.RSProcedureHandler(58): pid=34 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:24,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=34 java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:24,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=34 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:24,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/b47988fdf2664909b6524174dcd1de11 2024-11-21T03:40:24,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/e77cc8c7b4824fc78a5f8c0057623827 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/e77cc8c7b4824fc78a5f8c0057623827 2024-11-21T03:40:24,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/e77cc8c7b4824fc78a5f8c0057623827, entries=200, sequenceid=502, filesize=14.4 K 2024-11-21T03:40:24,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/36e3ca87482a4ff1b057797d1af0741d as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/36e3ca87482a4ff1b057797d1af0741d 2024-11-21T03:40:24,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/36e3ca87482a4ff1b057797d1af0741d, entries=100, sequenceid=502, filesize=9.6 K 2024-11-21T03:40:24,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/b47988fdf2664909b6524174dcd1de11 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/b47988fdf2664909b6524174dcd1de11 2024-11-21T03:40:24,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/b47988fdf2664909b6524174dcd1de11, entries=100, sequenceid=502, filesize=9.6 K 2024-11-21T03:40:24,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=53.67 KB/54960 for 30740eafbeb6526d1c29b0b1fec56da6 in 895ms, sequenceid=502, compaction requested=true 2024-11-21T03:40:24,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:24,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:40:24,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:24,374 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:24,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:40:24,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:24,374 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:24,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 30740eafbeb6526d1c29b0b1fec56da6:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:40:24,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:24,375 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35583 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:24,375 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40463 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:24,375 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/B is initiating minor compaction (all files) 2024-11-21T03:40:24,375 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/A is initiating minor compaction (all files) 2024-11-21T03:40:24,375 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/B in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:24,375 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/A in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:24,375 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/364faa70f8a34e4b9fafed70288da345, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/d17bfc64e3444d94bf8a930c5deb0b04, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/36e3ca87482a4ff1b057797d1af0741d] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=34.7 K 2024-11-21T03:40:24,375 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/8f7523b34fdd4a099d9406745e15d286, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/3c589201e9334e3caabcb4876030b219, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/e77cc8c7b4824fc78a5f8c0057623827] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=39.5 K 2024-11-21T03:40:24,376 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 364faa70f8a34e4b9fafed70288da345, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732160422227 2024-11-21T03:40:24,376 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f7523b34fdd4a099d9406745e15d286, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732160422227 2024-11-21T03:40:24,376 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting d17bfc64e3444d94bf8a930c5deb0b04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1732160422297 2024-11-21T03:40:24,376 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c589201e9334e3caabcb4876030b219, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1732160422297 2024-11-21T03:40:24,376 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 36e3ca87482a4ff1b057797d1af0741d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1732160423474 2024-11-21T03:40:24,376 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting e77cc8c7b4824fc78a5f8c0057623827, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1732160423470 2024-11-21T03:40:24,384 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#B#compaction#108 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:24,384 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#A#compaction#109 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:24,384 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/967b907f605a478baecc18bac2f7da95 is 50, key is test_row_0/A:col10/1732160423479/Put/seqid=0 2024-11-21T03:40:24,384 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/6d1d1f0363dd4b94bdf7bc9387d5d816 is 50, key is test_row_0/B:col10/1732160423479/Put/seqid=0 2024-11-21T03:40:24,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741945_1121 (size=13527) 2024-11-21T03:40:24,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741946_1122 (size=13527) 2024-11-21T03:40:24,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=34 2024-11-21T03:40:24,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:24,408 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:40:24,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/10791ca5405d456584d57c91c560d491 is 50, key is test_row_0/A:col10/1732160423490/Put/seqid=0 2024-11-21T03:40:24,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741947_1123 (size=12301) 2024-11-21T03:40:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-21T03:40:24,799 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/967b907f605a478baecc18bac2f7da95 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/967b907f605a478baecc18bac2f7da95 2024-11-21T03:40:24,799 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/6d1d1f0363dd4b94bdf7bc9387d5d816 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/6d1d1f0363dd4b94bdf7bc9387d5d816 2024-11-21T03:40:24,805 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/B of 30740eafbeb6526d1c29b0b1fec56da6 into 6d1d1f0363dd4b94bdf7bc9387d5d816(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:24,805 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/A of 30740eafbeb6526d1c29b0b1fec56da6 into 967b907f605a478baecc18bac2f7da95(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:24,805 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:24,805 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:24,805 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/B, priority=13, startTime=1732160424374; duration=0sec 2024-11-21T03:40:24,805 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/A, priority=13, startTime=1732160424374; duration=0sec 2024-11-21T03:40:24,805 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:24,805 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:B 2024-11-21T03:40:24,805 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:24,806 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:A 2024-11-21T03:40:24,806 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:24,807 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35583 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:24,807 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 30740eafbeb6526d1c29b0b1fec56da6/C is initiating minor compaction (all files) 2024-11-21T03:40:24,807 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 30740eafbeb6526d1c29b0b1fec56da6/C in TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:24,807 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/8dc74a722dd44836b72843166e4e7028, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0d2043db3546436a9ec0a71e4fe15cfa, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/b47988fdf2664909b6524174dcd1de11] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp, totalSize=34.7 K 2024-11-21T03:40:24,807 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dc74a722dd44836b72843166e4e7028, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1732160422227 2024-11-21T03:40:24,808 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d2043db3546436a9ec0a71e4fe15cfa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1732160422297 2024-11-21T03:40:24,808 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting b47988fdf2664909b6524174dcd1de11, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1732160423474 2024-11-21T03:40:24,815 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 30740eafbeb6526d1c29b0b1fec56da6#C#compaction#111 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:24,816 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/f274ea98d73449a4937f0e95adbda2a8 is 50, key is test_row_0/C:col10/1732160423479/Put/seqid=0 2024-11-21T03:40:24,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741948_1124 (size=13527) 2024-11-21T03:40:24,822 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/10791ca5405d456584d57c91c560d491 2024-11-21T03:40:24,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/03438521443f4a32a4b014628d2049f1 is 50, key is test_row_0/B:col10/1732160423490/Put/seqid=0 2024-11-21T03:40:24,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741949_1125 (size=12301) 2024-11-21T03:40:25,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:25,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. as already flushing 2024-11-21T03:40:25,099 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-21T03:40:25,099 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:40:25,099 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:25,099 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:25,099 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:40:25,191 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T03:40:25,232 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/f274ea98d73449a4937f0e95adbda2a8 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/f274ea98d73449a4937f0e95adbda2a8 2024-11-21T03:40:25,232 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/03438521443f4a32a4b014628d2049f1 2024-11-21T03:40:25,239 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 30740eafbeb6526d1c29b0b1fec56da6/C of 30740eafbeb6526d1c29b0b1fec56da6 into f274ea98d73449a4937f0e95adbda2a8(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:25,240 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:25,240 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., storeName=30740eafbeb6526d1c29b0b1fec56da6/C, priority=13, startTime=1732160424374; duration=0sec 2024-11-21T03:40:25,240 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:25,240 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 30740eafbeb6526d1c29b0b1fec56da6:C 2024-11-21T03:40:25,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=34}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/5071d48208df43a9afb344dd07971810 is 50, key is test_row_0/C:col10/1732160423490/Put/seqid=0 2024-11-21T03:40:25,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741950_1126 (size=12301) 2024-11-21T03:40:25,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-21T03:40:25,647 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/5071d48208df43a9afb344dd07971810 2024-11-21T03:40:25,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/10791ca5405d456584d57c91c560d491 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/10791ca5405d456584d57c91c560d491 2024-11-21T03:40:25,661 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/10791ca5405d456584d57c91c560d491, entries=150, sequenceid=515, filesize=12.0 K 2024-11-21T03:40:25,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/03438521443f4a32a4b014628d2049f1 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/03438521443f4a32a4b014628d2049f1 2024-11-21T03:40:25,666 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/03438521443f4a32a4b014628d2049f1, entries=150, sequenceid=515, filesize=12.0 K 2024-11-21T03:40:25,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/5071d48208df43a9afb344dd07971810 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5071d48208df43a9afb344dd07971810 2024-11-21T03:40:25,671 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5071d48208df43a9afb344dd07971810, entries=150, sequenceid=515, filesize=12.0 K 2024-11-21T03:40:25,672 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=6.71 KB/6870 for 30740eafbeb6526d1c29b0b1fec56da6 in 1264ms, sequenceid=515, compaction requested=false 2024-11-21T03:40:25,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.HRegion(2603): Flush status journal for 30740eafbeb6526d1c29b0b1fec56da6: 2024-11-21T03:40:25,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:25,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=34}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=34 2024-11-21T03:40:25,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=34 2024-11-21T03:40:25,675 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-21T03:40:25,675 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1930 sec 2024-11-21T03:40:25,676 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=33, table=TestAcidGuarantees in 2.1980 sec 2024-11-21T03:40:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-21T03:40:27,647 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:27,647 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T03:40:27,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-21T03:40:27,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-21T03:40:27,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-21T03:40:27,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-21T03:40:27,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-21T03:40:27,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T03:40:27,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3073 2024-11-21T03:40:27,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3139 2024-11-21T03:40:27,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T03:40:27,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1450 2024-11-21T03:40:27,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4350 rows 2024-11-21T03:40:27,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1463 2024-11-21T03:40:27,648 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4383 rows 2024-11-21T03:40:27,648 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T03:40:27,648 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T03:40:27,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:27,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:27,649 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:40:27,654 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T03:40:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T03:40:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-21T03:40:27,662 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732160427662"}]},"ts":"1732160427662"} 2024-11-21T03:40:27,664 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T03:40:27,665 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T03:40:27,666 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T03:40:27,670 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=30740eafbeb6526d1c29b0b1fec56da6, UNASSIGN}] 2024-11-21T03:40:27,671 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=30740eafbeb6526d1c29b0b1fec56da6, UNASSIGN 2024-11-21T03:40:27,672 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=30740eafbeb6526d1c29b0b1fec56da6, regionState=CLOSING, regionLocation=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:27,675 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=30740eafbeb6526d1c29b0b1fec56da6, UNASSIGN because future has completed 2024-11-21T03:40:27,675 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T03:40:27,675 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889}] 2024-11-21T03:40:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-21T03:40:27,832 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:27,832 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T03:40:27,833 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 30740eafbeb6526d1c29b0b1fec56da6, disabling compactions & flushes 2024-11-21T03:40:27,833 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:27,833 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:27,833 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. after waiting 0 ms 2024-11-21T03:40:27,833 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:27,833 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(2902): Flushing 30740eafbeb6526d1c29b0b1fec56da6 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-21T03:40:27,838 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/50d4752d872843baba0eb6a8ecdc1555 is 50, key is test_row_0/A:col10/1732160425097/Put/seqid=0 2024-11-21T03:40:27,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741951_1127 (size=7415) 2024-11-21T03:40:27,843 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/50d4752d872843baba0eb6a8ecdc1555 2024-11-21T03:40:27,851 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/35f769a80d02497ba5656ca28b2c4f50 is 50, key is test_row_0/B:col10/1732160425097/Put/seqid=0 2024-11-21T03:40:27,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741952_1128 (size=7415) 2024-11-21T03:40:27,859 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/35f769a80d02497ba5656ca28b2c4f50 2024-11-21T03:40:27,868 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/f866473dcdb84023b2e083473499f1b4 is 50, key is test_row_0/C:col10/1732160425097/Put/seqid=0 2024-11-21T03:40:27,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741953_1129 (size=7415) 2024-11-21T03:40:27,877 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=522 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/f866473dcdb84023b2e083473499f1b4 2024-11-21T03:40:27,882 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/A/50d4752d872843baba0eb6a8ecdc1555 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/50d4752d872843baba0eb6a8ecdc1555 2024-11-21T03:40:27,888 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/50d4752d872843baba0eb6a8ecdc1555, entries=50, sequenceid=522, filesize=7.2 K 2024-11-21T03:40:27,889 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/B/35f769a80d02497ba5656ca28b2c4f50 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/35f769a80d02497ba5656ca28b2c4f50 2024-11-21T03:40:27,896 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/35f769a80d02497ba5656ca28b2c4f50, entries=50, sequenceid=522, filesize=7.2 K 2024-11-21T03:40:27,897 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/.tmp/C/f866473dcdb84023b2e083473499f1b4 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/f866473dcdb84023b2e083473499f1b4 2024-11-21T03:40:27,902 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/f866473dcdb84023b2e083473499f1b4, entries=50, sequenceid=522, filesize=7.2 K 2024-11-21T03:40:27,903 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 30740eafbeb6526d1c29b0b1fec56da6 in 70ms, sequenceid=522, compaction requested=true 2024-11-21T03:40:27,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/365d827611094d0d881dea4b7dd607ad, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/398b337a12014e25ae5779af4272e34a, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/34491c071a41460abea92d377d14fcea, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/323c6912a05c45fd9f9089388d4f11a3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/ddd4610b0f7b4ce0a84f9470b5016587, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/044b2a01daaf4df0a374bdf160b256a8, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/cdf1a1eee8d645e79a625d341234ff9c, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/f059dcb9a100447ba39d447788c7f0cd, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/428fa94941b1435e85b6e2528e875336, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/46e61b69e4874d19a005b0f5528a98b1, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/e34a2b1612b14989b52791c23720b096, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/ef82e13423da4644ad1a73a1bd825e8e, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/de9572f63298471890ed5b972f5baa74, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/9814b5464c774bd893bc5451c4d9c911, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/78ae3972b89f43caaf9a7cdd51d2e94c, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/25cb959f983c435c994847ba3343a8f0, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/bcbab42508504d0a881299d502f86f5b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/32e713c260184cb98932fe87ab40905f, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/9b10cd2914bc4d928b82290dc314e5be, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/c1a3e6c3ae9a4a098a636e062c17774a, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/b7218ce9d4e14e7a881163b6650a5185, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/2916228537b34a78a4df97638781dfdd, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/3737ae57c77f4ddfac53de8e8aacd4e9, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/54f5fcf6560c496b8dd0dade23f72ba9, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/f65de732c5bc447b8138786d5138ace4, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/71adc4686bc146f595879a9d2b69021f, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/74b60654f895436aa97c0a2aeb7dfc6f, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/107da299bafd482bb2af07da902eb880, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/c1864d5954a044ea8969cb2c7160d829, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/8a6b1732378b4381861cc89279102cf3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/315f3b87e0d849c5be9a13aad60f386b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/7e597a0377234bc8bf93b6ad4048367d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/a4cf434374d647b58abcdbb9dc110fee, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/8f7523b34fdd4a099d9406745e15d286, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/3c589201e9334e3caabcb4876030b219, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/e77cc8c7b4824fc78a5f8c0057623827] to archive 2024-11-21T03:40:27,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:40:27,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/365d827611094d0d881dea4b7dd607ad to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/365d827611094d0d881dea4b7dd607ad 2024-11-21T03:40:27,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/398b337a12014e25ae5779af4272e34a to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/398b337a12014e25ae5779af4272e34a 2024-11-21T03:40:27,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/34491c071a41460abea92d377d14fcea to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/34491c071a41460abea92d377d14fcea 2024-11-21T03:40:27,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/323c6912a05c45fd9f9089388d4f11a3 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/323c6912a05c45fd9f9089388d4f11a3 2024-11-21T03:40:27,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/ddd4610b0f7b4ce0a84f9470b5016587 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/ddd4610b0f7b4ce0a84f9470b5016587 2024-11-21T03:40:27,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/044b2a01daaf4df0a374bdf160b256a8 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/044b2a01daaf4df0a374bdf160b256a8 2024-11-21T03:40:27,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/cdf1a1eee8d645e79a625d341234ff9c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/cdf1a1eee8d645e79a625d341234ff9c 2024-11-21T03:40:27,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/f059dcb9a100447ba39d447788c7f0cd to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/f059dcb9a100447ba39d447788c7f0cd 2024-11-21T03:40:27,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/428fa94941b1435e85b6e2528e875336 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/428fa94941b1435e85b6e2528e875336 2024-11-21T03:40:27,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/46e61b69e4874d19a005b0f5528a98b1 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/46e61b69e4874d19a005b0f5528a98b1 2024-11-21T03:40:27,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/e34a2b1612b14989b52791c23720b096 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/e34a2b1612b14989b52791c23720b096 2024-11-21T03:40:27,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/ef82e13423da4644ad1a73a1bd825e8e to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/ef82e13423da4644ad1a73a1bd825e8e 2024-11-21T03:40:27,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/de9572f63298471890ed5b972f5baa74 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/de9572f63298471890ed5b972f5baa74 2024-11-21T03:40:27,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/9814b5464c774bd893bc5451c4d9c911 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/9814b5464c774bd893bc5451c4d9c911 2024-11-21T03:40:27,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/78ae3972b89f43caaf9a7cdd51d2e94c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/78ae3972b89f43caaf9a7cdd51d2e94c 2024-11-21T03:40:27,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/25cb959f983c435c994847ba3343a8f0 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/25cb959f983c435c994847ba3343a8f0 2024-11-21T03:40:27,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/bcbab42508504d0a881299d502f86f5b to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/bcbab42508504d0a881299d502f86f5b 2024-11-21T03:40:27,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/32e713c260184cb98932fe87ab40905f to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/32e713c260184cb98932fe87ab40905f 2024-11-21T03:40:27,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/9b10cd2914bc4d928b82290dc314e5be to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/9b10cd2914bc4d928b82290dc314e5be 2024-11-21T03:40:27,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/c1a3e6c3ae9a4a098a636e062c17774a to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/c1a3e6c3ae9a4a098a636e062c17774a 2024-11-21T03:40:27,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/b7218ce9d4e14e7a881163b6650a5185 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/b7218ce9d4e14e7a881163b6650a5185 2024-11-21T03:40:27,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/2916228537b34a78a4df97638781dfdd to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/2916228537b34a78a4df97638781dfdd 2024-11-21T03:40:27,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/3737ae57c77f4ddfac53de8e8aacd4e9 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/3737ae57c77f4ddfac53de8e8aacd4e9 2024-11-21T03:40:27,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/54f5fcf6560c496b8dd0dade23f72ba9 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/54f5fcf6560c496b8dd0dade23f72ba9 2024-11-21T03:40:27,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/f65de732c5bc447b8138786d5138ace4 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/f65de732c5bc447b8138786d5138ace4 2024-11-21T03:40:27,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/71adc4686bc146f595879a9d2b69021f to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/71adc4686bc146f595879a9d2b69021f 2024-11-21T03:40:27,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/74b60654f895436aa97c0a2aeb7dfc6f to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/74b60654f895436aa97c0a2aeb7dfc6f 2024-11-21T03:40:27,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/107da299bafd482bb2af07da902eb880 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/107da299bafd482bb2af07da902eb880 2024-11-21T03:40:27,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/c1864d5954a044ea8969cb2c7160d829 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/c1864d5954a044ea8969cb2c7160d829 2024-11-21T03:40:27,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/8a6b1732378b4381861cc89279102cf3 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/8a6b1732378b4381861cc89279102cf3 2024-11-21T03:40:27,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/315f3b87e0d849c5be9a13aad60f386b to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/315f3b87e0d849c5be9a13aad60f386b 2024-11-21T03:40:27,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/7e597a0377234bc8bf93b6ad4048367d to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/7e597a0377234bc8bf93b6ad4048367d 2024-11-21T03:40:27,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/a4cf434374d647b58abcdbb9dc110fee to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/a4cf434374d647b58abcdbb9dc110fee 2024-11-21T03:40:27,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/8f7523b34fdd4a099d9406745e15d286 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/8f7523b34fdd4a099d9406745e15d286 2024-11-21T03:40:27,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/3c589201e9334e3caabcb4876030b219 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/3c589201e9334e3caabcb4876030b219 2024-11-21T03:40:27,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/e77cc8c7b4824fc78a5f8c0057623827 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/e77cc8c7b4824fc78a5f8c0057623827 2024-11-21T03:40:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-21T03:40:27,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/d70061ed7b8e45e294b1b68c0e28ac2d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/8e8dccb3c25d4c5d8e09c07a654258c0, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/cf08d99cccab47bb816041b251243952, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/755b234385d0459880f7ca266992feab, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/5fc193f8f23545de9502092eabef0b09, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/7caa146aab8c4aa9b527e2fb27e13932, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/beeaf37cd645479fa79c3e31c9e97c6c, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/f54799c1f0594f4e9d23473d38743232, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/14a3648a305c4737a9b4e82c55cd7891, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/3b3f9be376cb42fdbc94865993799577, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/aeba7022284546979ae9f9dcee48c520, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/96f77690e726435db394d700776af692, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/ed07bd9f370543229293f566a1c2f4e4, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/8ffcf7cc8bdf41d989b42c376cd4889b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/3c219f366fc94025a61d319f970789fd, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/a83f2f701e174734ad546b5b3c938a5e, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/c524b661ad2e453499abdb2fffed041a, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/165709d9756845cd8758774f0f7b9c66, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/b23c401d49c0401e8b51b57401663d37, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/cccad8c482b144dc8951de7d6ef2014a, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/52d79324325c469f866a02dddbd92fd6, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/6783128a498f40cdace42808594f1737, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/37b68a6c2aad4f95a781f6fc8d48d8d7, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/bf7e1223167140959638620c1d5d3dd1, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/7a143d14bc484f10a38590a53897ac62, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/4d201343934e42c2b1f9b8e9e2a10f59, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/9c8960a12bfe4616babdca6ca8e9481c, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/0584f330b88b4db2b331f794614c91d6, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/49c958c785964d81b6188b131e1dd62b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/499ec27657e54e4b90ba430b79e08b4f, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/b0845b6f1a104ac19c36c5822527981d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/7f62a254b6c242a1aa18fe6009a9102f, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/364faa70f8a34e4b9fafed70288da345, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/ac3e9679fa4a4f98abdb636c0d5094d3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/d17bfc64e3444d94bf8a930c5deb0b04, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/36e3ca87482a4ff1b057797d1af0741d] to archive 2024-11-21T03:40:27,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:40:27,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/d70061ed7b8e45e294b1b68c0e28ac2d to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/d70061ed7b8e45e294b1b68c0e28ac2d 2024-11-21T03:40:27,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/8e8dccb3c25d4c5d8e09c07a654258c0 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/8e8dccb3c25d4c5d8e09c07a654258c0 2024-11-21T03:40:27,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/cf08d99cccab47bb816041b251243952 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/cf08d99cccab47bb816041b251243952 2024-11-21T03:40:27,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/755b234385d0459880f7ca266992feab to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/755b234385d0459880f7ca266992feab 2024-11-21T03:40:27,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/5fc193f8f23545de9502092eabef0b09 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/5fc193f8f23545de9502092eabef0b09 2024-11-21T03:40:27,993 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/7caa146aab8c4aa9b527e2fb27e13932 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/7caa146aab8c4aa9b527e2fb27e13932 2024-11-21T03:40:27,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/beeaf37cd645479fa79c3e31c9e97c6c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/beeaf37cd645479fa79c3e31c9e97c6c 2024-11-21T03:40:27,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/f54799c1f0594f4e9d23473d38743232 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/f54799c1f0594f4e9d23473d38743232 2024-11-21T03:40:27,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/14a3648a305c4737a9b4e82c55cd7891 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/14a3648a305c4737a9b4e82c55cd7891 2024-11-21T03:40:27,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/3b3f9be376cb42fdbc94865993799577 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/3b3f9be376cb42fdbc94865993799577 2024-11-21T03:40:28,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/aeba7022284546979ae9f9dcee48c520 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/aeba7022284546979ae9f9dcee48c520 2024-11-21T03:40:28,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/96f77690e726435db394d700776af692 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/96f77690e726435db394d700776af692 2024-11-21T03:40:28,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/ed07bd9f370543229293f566a1c2f4e4 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/ed07bd9f370543229293f566a1c2f4e4 2024-11-21T03:40:28,004 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/8ffcf7cc8bdf41d989b42c376cd4889b to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/8ffcf7cc8bdf41d989b42c376cd4889b 2024-11-21T03:40:28,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/3c219f366fc94025a61d319f970789fd to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/3c219f366fc94025a61d319f970789fd 2024-11-21T03:40:28,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/a83f2f701e174734ad546b5b3c938a5e to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/a83f2f701e174734ad546b5b3c938a5e 2024-11-21T03:40:28,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/c524b661ad2e453499abdb2fffed041a to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/c524b661ad2e453499abdb2fffed041a 2024-11-21T03:40:28,008 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/165709d9756845cd8758774f0f7b9c66 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/165709d9756845cd8758774f0f7b9c66 2024-11-21T03:40:28,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/b23c401d49c0401e8b51b57401663d37 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/b23c401d49c0401e8b51b57401663d37 2024-11-21T03:40:28,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/cccad8c482b144dc8951de7d6ef2014a to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/cccad8c482b144dc8951de7d6ef2014a 2024-11-21T03:40:28,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/52d79324325c469f866a02dddbd92fd6 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/52d79324325c469f866a02dddbd92fd6 2024-11-21T03:40:28,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/6783128a498f40cdace42808594f1737 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/6783128a498f40cdace42808594f1737 2024-11-21T03:40:28,015 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/37b68a6c2aad4f95a781f6fc8d48d8d7 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/37b68a6c2aad4f95a781f6fc8d48d8d7 2024-11-21T03:40:28,016 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/bf7e1223167140959638620c1d5d3dd1 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/bf7e1223167140959638620c1d5d3dd1 2024-11-21T03:40:28,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/7a143d14bc484f10a38590a53897ac62 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/7a143d14bc484f10a38590a53897ac62 2024-11-21T03:40:28,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/4d201343934e42c2b1f9b8e9e2a10f59 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/4d201343934e42c2b1f9b8e9e2a10f59 2024-11-21T03:40:28,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/9c8960a12bfe4616babdca6ca8e9481c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/9c8960a12bfe4616babdca6ca8e9481c 2024-11-21T03:40:28,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/0584f330b88b4db2b331f794614c91d6 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/0584f330b88b4db2b331f794614c91d6 2024-11-21T03:40:28,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/49c958c785964d81b6188b131e1dd62b to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/49c958c785964d81b6188b131e1dd62b 2024-11-21T03:40:28,025 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/499ec27657e54e4b90ba430b79e08b4f to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/499ec27657e54e4b90ba430b79e08b4f 2024-11-21T03:40:28,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/b0845b6f1a104ac19c36c5822527981d to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/b0845b6f1a104ac19c36c5822527981d 2024-11-21T03:40:28,028 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/7f62a254b6c242a1aa18fe6009a9102f to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/7f62a254b6c242a1aa18fe6009a9102f 2024-11-21T03:40:28,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/364faa70f8a34e4b9fafed70288da345 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/364faa70f8a34e4b9fafed70288da345 2024-11-21T03:40:28,031 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/ac3e9679fa4a4f98abdb636c0d5094d3 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/ac3e9679fa4a4f98abdb636c0d5094d3 2024-11-21T03:40:28,032 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/d17bfc64e3444d94bf8a930c5deb0b04 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/d17bfc64e3444d94bf8a930c5deb0b04 2024-11-21T03:40:28,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/36e3ca87482a4ff1b057797d1af0741d to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/36e3ca87482a4ff1b057797d1af0741d 2024-11-21T03:40:28,034 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5a58e705027b4a5da9727adc20fe6541, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/1bb1422771a54839ba7ad2f83b1bb0e1, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/cc963af1f83b4b13b58d4e2347a4252c, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/b350a51564a64d6dad5ea6fcd9cd5c00, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/685674586cb54b9c9ec8d71ca8e98c68, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5b0142154e2c44a2a048d4115ee2a7a0, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/be17df8fa86f4ed980fa5c769653dc26, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/75c1e6c8a3824e5f90441398a9dd65c9, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5736edc2061b44d1a63ab1013ce2274d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0aa9c4640b774cba9f501279ca734a9c, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/6137c643a72549f4bd00b1066cb8e1e3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/bbfb949d9d30470599a79becb807878c, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/72281ac7c9f14f3b94dbcb63d84ba956, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/3bd5fe1a466742959222d4a46f38dcfd, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/f05845015b7b42cdb10eab617a3787d1, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/ecde05e792464c9eb08255ec8181310a, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0d57504694d44965b3c1ba3963b477d5, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/1fd222afa8a84b67a6aa7ede3b086545, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/8a2c7aa24ba144c1bf3c14ed68e8651a, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/02650184ac804c118a34d7227dfc6028, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/519d14b4b3ba4ddb9dba09a2bdcd8ff4, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0c2c359798144025aa658127b0fd3fd0, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/05f7812cfa8146228675dcf5a100c12b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/c1525cbf57dc44679d96f60db735bcff, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/20652d2532e448b282d28e98ca82af4e, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/7e5e20f9e01e4aeea6647310c3008396, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/c1206659a26749c4938395f436953364, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0ba37b2805a54de2bd1728fb60763874, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/7350291e272c42d0b640cf2735932342, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/b71c77d1ba8e4bf08c2998c398d40ca2, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5e461e6369cd4acfaf5ad659ea77f896, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/852bd229b95d4ea4a2d516c6426615e3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/8dc74a722dd44836b72843166e4e7028, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/e3dbbb1e8f164c1e8e4746b4d487f502, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0d2043db3546436a9ec0a71e4fe15cfa, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/b47988fdf2664909b6524174dcd1de11] to archive 2024-11-21T03:40:28,036 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:40:28,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5a58e705027b4a5da9727adc20fe6541 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5a58e705027b4a5da9727adc20fe6541 2024-11-21T03:40:28,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/1bb1422771a54839ba7ad2f83b1bb0e1 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/1bb1422771a54839ba7ad2f83b1bb0e1 2024-11-21T03:40:28,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/cc963af1f83b4b13b58d4e2347a4252c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/cc963af1f83b4b13b58d4e2347a4252c 2024-11-21T03:40:28,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/b350a51564a64d6dad5ea6fcd9cd5c00 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/b350a51564a64d6dad5ea6fcd9cd5c00 2024-11-21T03:40:28,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/685674586cb54b9c9ec8d71ca8e98c68 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/685674586cb54b9c9ec8d71ca8e98c68 2024-11-21T03:40:28,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5b0142154e2c44a2a048d4115ee2a7a0 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5b0142154e2c44a2a048d4115ee2a7a0 2024-11-21T03:40:28,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/be17df8fa86f4ed980fa5c769653dc26 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/be17df8fa86f4ed980fa5c769653dc26 2024-11-21T03:40:28,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/75c1e6c8a3824e5f90441398a9dd65c9 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/75c1e6c8a3824e5f90441398a9dd65c9 2024-11-21T03:40:28,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5736edc2061b44d1a63ab1013ce2274d to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5736edc2061b44d1a63ab1013ce2274d 2024-11-21T03:40:28,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0aa9c4640b774cba9f501279ca734a9c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0aa9c4640b774cba9f501279ca734a9c 2024-11-21T03:40:28,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/6137c643a72549f4bd00b1066cb8e1e3 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/6137c643a72549f4bd00b1066cb8e1e3 2024-11-21T03:40:28,049 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/bbfb949d9d30470599a79becb807878c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/bbfb949d9d30470599a79becb807878c 2024-11-21T03:40:28,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/72281ac7c9f14f3b94dbcb63d84ba956 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/72281ac7c9f14f3b94dbcb63d84ba956 2024-11-21T03:40:28,052 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/3bd5fe1a466742959222d4a46f38dcfd to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/3bd5fe1a466742959222d4a46f38dcfd 2024-11-21T03:40:28,053 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/f05845015b7b42cdb10eab617a3787d1 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/f05845015b7b42cdb10eab617a3787d1 2024-11-21T03:40:28,055 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/ecde05e792464c9eb08255ec8181310a to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/ecde05e792464c9eb08255ec8181310a 2024-11-21T03:40:28,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0d57504694d44965b3c1ba3963b477d5 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0d57504694d44965b3c1ba3963b477d5 2024-11-21T03:40:28,057 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/1fd222afa8a84b67a6aa7ede3b086545 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/1fd222afa8a84b67a6aa7ede3b086545 2024-11-21T03:40:28,059 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/8a2c7aa24ba144c1bf3c14ed68e8651a to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/8a2c7aa24ba144c1bf3c14ed68e8651a 2024-11-21T03:40:28,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/02650184ac804c118a34d7227dfc6028 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/02650184ac804c118a34d7227dfc6028 2024-11-21T03:40:28,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/519d14b4b3ba4ddb9dba09a2bdcd8ff4 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/519d14b4b3ba4ddb9dba09a2bdcd8ff4 2024-11-21T03:40:28,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0c2c359798144025aa658127b0fd3fd0 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0c2c359798144025aa658127b0fd3fd0 2024-11-21T03:40:28,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/05f7812cfa8146228675dcf5a100c12b to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/05f7812cfa8146228675dcf5a100c12b 2024-11-21T03:40:28,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/c1525cbf57dc44679d96f60db735bcff to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/c1525cbf57dc44679d96f60db735bcff 2024-11-21T03:40:28,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/20652d2532e448b282d28e98ca82af4e to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/20652d2532e448b282d28e98ca82af4e 2024-11-21T03:40:28,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/7e5e20f9e01e4aeea6647310c3008396 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/7e5e20f9e01e4aeea6647310c3008396 2024-11-21T03:40:28,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/c1206659a26749c4938395f436953364 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/c1206659a26749c4938395f436953364 2024-11-21T03:40:28,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0ba37b2805a54de2bd1728fb60763874 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0ba37b2805a54de2bd1728fb60763874 2024-11-21T03:40:28,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/7350291e272c42d0b640cf2735932342 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/7350291e272c42d0b640cf2735932342 2024-11-21T03:40:28,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/b71c77d1ba8e4bf08c2998c398d40ca2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/b71c77d1ba8e4bf08c2998c398d40ca2 2024-11-21T03:40:28,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5e461e6369cd4acfaf5ad659ea77f896 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5e461e6369cd4acfaf5ad659ea77f896 2024-11-21T03:40:28,075 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/852bd229b95d4ea4a2d516c6426615e3 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/852bd229b95d4ea4a2d516c6426615e3 2024-11-21T03:40:28,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/8dc74a722dd44836b72843166e4e7028 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/8dc74a722dd44836b72843166e4e7028 2024-11-21T03:40:28,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/e3dbbb1e8f164c1e8e4746b4d487f502 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/e3dbbb1e8f164c1e8e4746b4d487f502 2024-11-21T03:40:28,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0d2043db3546436a9ec0a71e4fe15cfa to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/0d2043db3546436a9ec0a71e4fe15cfa 2024-11-21T03:40:28,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/b47988fdf2664909b6524174dcd1de11 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/b47988fdf2664909b6524174dcd1de11 2024-11-21T03:40:28,086 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/recovered.edits/525.seqid, newMaxSeqId=525, maxSeqId=1 2024-11-21T03:40:28,087 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. 2024-11-21T03:40:28,087 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 30740eafbeb6526d1c29b0b1fec56da6: Waiting for close lock at 1732160427833Running coprocessor pre-close hooks at 1732160427833Disabling compacts and flushes for region at 1732160427833Disabling writes for close at 1732160427833Obtaining lock to block concurrent updates at 1732160427833Preparing flush snapshotting stores in 30740eafbeb6526d1c29b0b1fec56da6 at 1732160427833Finished memstore snapshotting TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=18720, getOffHeapSize=0, getCellsCount=150 at 1732160427834 (+1 ms)Flushing stores of TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6. at 1732160427834Flushing 30740eafbeb6526d1c29b0b1fec56da6/A: creating writer at 1732160427834Flushing 30740eafbeb6526d1c29b0b1fec56da6/A: appending metadata at 1732160427837 (+3 ms)Flushing 30740eafbeb6526d1c29b0b1fec56da6/A: closing flushed file at 1732160427837Flushing 30740eafbeb6526d1c29b0b1fec56da6/B: creating writer at 1732160427848 (+11 ms)Flushing 30740eafbeb6526d1c29b0b1fec56da6/B: appending metadata at 1732160427851 (+3 ms)Flushing 30740eafbeb6526d1c29b0b1fec56da6/B: closing flushed file at 1732160427851Flushing 30740eafbeb6526d1c29b0b1fec56da6/C: creating writer at 1732160427864 (+13 ms)Flushing 30740eafbeb6526d1c29b0b1fec56da6/C: appending metadata at 1732160427867 (+3 ms)Flushing 30740eafbeb6526d1c29b0b1fec56da6/C: closing flushed file at 1732160427867Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13d90230: reopening flushed file at 1732160427881 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71a2476: reopening flushed file at 1732160427888 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5073a19c: reopening flushed file at 1732160427896 (+8 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 30740eafbeb6526d1c29b0b1fec56da6 in 70ms, sequenceid=522, compaction requested=true at 1732160427903 (+7 ms)Writing region close event to WAL at 1732160428082 (+179 ms)Running coprocessor post-close hooks at 1732160428087 (+5 ms)Closed at 1732160428087 2024-11-21T03:40:28,090 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:28,091 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=30740eafbeb6526d1c29b0b1fec56da6, regionState=CLOSED 2024-11-21T03:40:28,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 because future has completed 2024-11-21T03:40:28,099 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-21T03:40:28,099 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 30740eafbeb6526d1c29b0b1fec56da6, server=fdc0a393186c,34631,1732160396889 in 422 msec 2024-11-21T03:40:28,101 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-21T03:40:28,101 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=30740eafbeb6526d1c29b0b1fec56da6, UNASSIGN in 429 msec 2024-11-21T03:40:28,103 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-21T03:40:28,104 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 436 msec 2024-11-21T03:40:28,105 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732160428105"}]},"ts":"1732160428105"} 2024-11-21T03:40:28,106 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T03:40:28,106 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T03:40:28,108 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 451 msec 2024-11-21T03:40:28,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-21T03:40:28,288 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:28,292 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T03:40:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:40:28,299 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=39, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:40:28,301 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=39, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:40:28,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-21T03:40:28,307 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:28,312 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A, FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B, FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C, FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/recovered.edits] 2024-11-21T03:40:28,316 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/10791ca5405d456584d57c91c560d491 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/10791ca5405d456584d57c91c560d491 2024-11-21T03:40:28,317 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/50d4752d872843baba0eb6a8ecdc1555 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/50d4752d872843baba0eb6a8ecdc1555 2024-11-21T03:40:28,319 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/967b907f605a478baecc18bac2f7da95 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/A/967b907f605a478baecc18bac2f7da95 2024-11-21T03:40:28,321 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/03438521443f4a32a4b014628d2049f1 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/03438521443f4a32a4b014628d2049f1 2024-11-21T03:40:28,322 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/35f769a80d02497ba5656ca28b2c4f50 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/35f769a80d02497ba5656ca28b2c4f50 2024-11-21T03:40:28,324 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/6d1d1f0363dd4b94bdf7bc9387d5d816 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/B/6d1d1f0363dd4b94bdf7bc9387d5d816 2024-11-21T03:40:28,326 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5071d48208df43a9afb344dd07971810 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/5071d48208df43a9afb344dd07971810 2024-11-21T03:40:28,327 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/f274ea98d73449a4937f0e95adbda2a8 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/f274ea98d73449a4937f0e95adbda2a8 2024-11-21T03:40:28,328 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/f866473dcdb84023b2e083473499f1b4 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/C/f866473dcdb84023b2e083473499f1b4 2024-11-21T03:40:28,331 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/recovered.edits/525.seqid to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6/recovered.edits/525.seqid 2024-11-21T03:40:28,332 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/30740eafbeb6526d1c29b0b1fec56da6 2024-11-21T03:40:28,332 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T03:40:28,337 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=39, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:40:28,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-21T03:40:28,345 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T03:40:28,379 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T03:40:28,380 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=39, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:40:28,381 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T03:40:28,381 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732160428381"}]},"ts":"9223372036854775807"} 2024-11-21T03:40:28,387 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T03:40:28,388 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 30740eafbeb6526d1c29b0b1fec56da6, NAME => 'TestAcidGuarantees,,1732160401216.30740eafbeb6526d1c29b0b1fec56da6.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T03:40:28,388 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T03:40:28,388 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732160428388"}]},"ts":"9223372036854775807"} 2024-11-21T03:40:28,392 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T03:40:28,393 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=39, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:40:28,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 100 msec 2024-11-21T03:40:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-11-21T03:40:28,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T03:40:28,408 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:28,424 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=239 (was 211) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;fdc0a393186c:34631-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_263764961_22 at /127.0.0.1:45966 [Waiting for operation #315] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-677494979_22 at /127.0.0.1:58994 [Waiting for operation #329] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_263764961_22 at /127.0.0.1:51718 [Waiting for operation #78] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=521 (was 388) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7205 (was 7264) 2024-11-21T03:40:28,436 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=521, ProcessCount=11, AvailableMemoryMB=7202 2024-11-21T03:40:28,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T03:40:28,438 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T03:40:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T03:40:28,441 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=40, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T03:40:28,441 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:28,441 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 40 2024-11-21T03:40:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T03:40:28,442 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=40, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T03:40:28,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741954_1130 (size=959) 2024-11-21T03:40:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T03:40:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T03:40:28,852 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0 2024-11-21T03:40:28,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741955_1131 (size=53) 2024-11-21T03:40:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T03:40:29,263 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:40:29,264 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 9a822f4697b07ce815f004ac9ea14bb2, disabling compactions & flushes 2024-11-21T03:40:29,264 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:29,264 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:29,264 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. after waiting 0 ms 2024-11-21T03:40:29,264 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:29,264 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:29,264 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9a822f4697b07ce815f004ac9ea14bb2: Waiting for close lock at 1732160429263Disabling compacts and flushes for region at 1732160429263Disabling writes for close at 1732160429264 (+1 ms)Writing region close event to WAL at 1732160429264Closed at 1732160429264 2024-11-21T03:40:29,267 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=40, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T03:40:29,267 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732160429267"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732160429267"}]},"ts":"1732160429267"} 2024-11-21T03:40:29,271 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T03:40:29,272 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=40, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T03:40:29,272 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732160429272"}]},"ts":"1732160429272"} 2024-11-21T03:40:29,273 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T03:40:29,274 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a822f4697b07ce815f004ac9ea14bb2, ASSIGN}] 2024-11-21T03:40:29,275 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a822f4697b07ce815f004ac9ea14bb2, ASSIGN 2024-11-21T03:40:29,276 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a822f4697b07ce815f004ac9ea14bb2, ASSIGN; state=OFFLINE, location=fdc0a393186c,34631,1732160396889; forceNewPlan=false, retain=false 2024-11-21T03:40:29,426 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=9a822f4697b07ce815f004ac9ea14bb2, regionState=OPENING, regionLocation=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:29,429 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=40, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a822f4697b07ce815f004ac9ea14bb2, ASSIGN because future has completed 2024-11-21T03:40:29,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889}] 2024-11-21T03:40:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T03:40:29,585 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:29,585 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:40:29,586 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:29,586 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:40:29,586 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:29,586 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:29,587 INFO [StoreOpener-9a822f4697b07ce815f004ac9ea14bb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:29,589 INFO [StoreOpener-9a822f4697b07ce815f004ac9ea14bb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a822f4697b07ce815f004ac9ea14bb2 columnFamilyName A 2024-11-21T03:40:29,589 DEBUG [StoreOpener-9a822f4697b07ce815f004ac9ea14bb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:29,590 INFO [StoreOpener-9a822f4697b07ce815f004ac9ea14bb2-1 {}] regionserver.HStore(327): Store=9a822f4697b07ce815f004ac9ea14bb2/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:40:29,590 INFO [StoreOpener-9a822f4697b07ce815f004ac9ea14bb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:29,591 INFO [StoreOpener-9a822f4697b07ce815f004ac9ea14bb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a822f4697b07ce815f004ac9ea14bb2 columnFamilyName B 2024-11-21T03:40:29,591 DEBUG [StoreOpener-9a822f4697b07ce815f004ac9ea14bb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:29,592 INFO [StoreOpener-9a822f4697b07ce815f004ac9ea14bb2-1 {}] regionserver.HStore(327): Store=9a822f4697b07ce815f004ac9ea14bb2/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:40:29,592 INFO [StoreOpener-9a822f4697b07ce815f004ac9ea14bb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:29,593 INFO [StoreOpener-9a822f4697b07ce815f004ac9ea14bb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a822f4697b07ce815f004ac9ea14bb2 columnFamilyName C 2024-11-21T03:40:29,593 DEBUG [StoreOpener-9a822f4697b07ce815f004ac9ea14bb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:29,593 INFO [StoreOpener-9a822f4697b07ce815f004ac9ea14bb2-1 {}] regionserver.HStore(327): Store=9a822f4697b07ce815f004ac9ea14bb2/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:40:29,594 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:29,594 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:29,595 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:29,595 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:29,596 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:29,596 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T03:40:29,597 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:29,600 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T03:40:29,600 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 9a822f4697b07ce815f004ac9ea14bb2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69086277, jitterRate=0.029465749859809875}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T03:40:29,600 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:29,601 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 9a822f4697b07ce815f004ac9ea14bb2: Running coprocessor pre-open hook at 1732160429586Writing region info on filesystem at 1732160429586Initializing all the Stores at 1732160429587 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732160429587Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732160429587Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732160429587Cleaning up temporary data from old regions at 1732160429596 (+9 ms)Running coprocessor post-open hooks at 1732160429600 (+4 ms)Region opened successfully at 1732160429601 (+1 ms) 2024-11-21T03:40:29,602 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., pid=42, masterSystemTime=1732160429582 2024-11-21T03:40:29,604 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:29,604 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:29,604 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=41 updating hbase:meta row=9a822f4697b07ce815f004ac9ea14bb2, regionState=OPEN, openSeqNum=2, regionLocation=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:29,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=41, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 because future has completed 2024-11-21T03:40:29,609 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=41 2024-11-21T03:40:29,609 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=41, state=SUCCESS, hasLock=false; OpenRegionProcedure 9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 in 177 msec 2024-11-21T03:40:29,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-21T03:40:29,612 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a822f4697b07ce815f004ac9ea14bb2, ASSIGN in 335 msec 2024-11-21T03:40:29,613 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=40, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T03:40:29,613 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732160429613"}]},"ts":"1732160429613"} 2024-11-21T03:40:29,615 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T03:40:29,616 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=40, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T03:40:29,617 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1780 sec 2024-11-21T03:40:30,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T03:40:30,597 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:30,598 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bcb2494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:30,598 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:30,598 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:30,600 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:30,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:30,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:30,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f024497, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:30,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:30,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:30,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:30,602 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40300, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:30,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f289009, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:30,603 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:30,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:30,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:30,606 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:30,608 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T03:40:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is fdc0a393186c,44285,1732160396080 2024-11-21T03:40:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@281d24f1 2024-11-21T03:40:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T03:40:30,610 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T03:40:30,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T03:40:30,614 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T03:40:30,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=43, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-21T03:40:30,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741956_1132 (size=995) 2024-11-21T03:40:31,043 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-21T03:40:31,043 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-21T03:40:31,048 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T03:40:31,056 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a822f4697b07ce815f004ac9ea14bb2, REOPEN/MOVE}] 2024-11-21T03:40:31,056 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a822f4697b07ce815f004ac9ea14bb2, REOPEN/MOVE 2024-11-21T03:40:31,057 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=45 updating hbase:meta row=9a822f4697b07ce815f004ac9ea14bb2, regionState=CLOSING, regionLocation=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:31,058 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a822f4697b07ce815f004ac9ea14bb2, REOPEN/MOVE because future has completed 2024-11-21T03:40:31,059 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T03:40:31,059 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889}] 2024-11-21T03:40:31,212 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(122): Close 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:31,212 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T03:40:31,212 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1722): Closing 9a822f4697b07ce815f004ac9ea14bb2, disabling compactions & flushes 2024-11-21T03:40:31,212 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:31,212 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:31,212 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. after waiting 0 ms 2024-11-21T03:40:31,212 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:31,216 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T03:40:31,217 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:31,217 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1676): Region close journal for 9a822f4697b07ce815f004ac9ea14bb2: Waiting for close lock at 1732160431212Running coprocessor pre-close hooks at 1732160431212Disabling compacts and flushes for region at 1732160431212Disabling writes for close at 1732160431212Writing region close event to WAL at 1732160431213 (+1 ms)Running coprocessor post-close hooks at 1732160431217 (+4 ms)Closed at 1732160431217 2024-11-21T03:40:31,217 WARN [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegionServer(3299): Not adding moved region record: 9a822f4697b07ce815f004ac9ea14bb2 to self. 2024-11-21T03:40:31,219 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(157): Closed 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:31,220 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=45 updating hbase:meta row=9a822f4697b07ce815f004ac9ea14bb2, regionState=CLOSED 2024-11-21T03:40:31,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=45, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 because future has completed 2024-11-21T03:40:31,224 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=45 2024-11-21T03:40:31,225 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=45, state=SUCCESS, hasLock=false; CloseRegionProcedure 9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 in 164 msec 2024-11-21T03:40:31,225 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a822f4697b07ce815f004ac9ea14bb2, REOPEN/MOVE; state=CLOSED, location=fdc0a393186c,34631,1732160396889; forceNewPlan=false, retain=true 2024-11-21T03:40:31,376 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=45 updating hbase:meta row=9a822f4697b07ce815f004ac9ea14bb2, regionState=OPENING, regionLocation=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:31,378 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a822f4697b07ce815f004ac9ea14bb2, REOPEN/MOVE because future has completed 2024-11-21T03:40:31,378 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=45, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889}] 2024-11-21T03:40:31,534 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:31,534 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:40:31,535 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:31,535 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:40:31,535 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:31,535 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:31,540 INFO [StoreOpener-9a822f4697b07ce815f004ac9ea14bb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:31,548 INFO [StoreOpener-9a822f4697b07ce815f004ac9ea14bb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a822f4697b07ce815f004ac9ea14bb2 columnFamilyName A 2024-11-21T03:40:31,549 DEBUG [StoreOpener-9a822f4697b07ce815f004ac9ea14bb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:31,550 INFO [StoreOpener-9a822f4697b07ce815f004ac9ea14bb2-1 {}] regionserver.HStore(327): Store=9a822f4697b07ce815f004ac9ea14bb2/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:40:31,550 INFO [StoreOpener-9a822f4697b07ce815f004ac9ea14bb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:31,551 INFO [StoreOpener-9a822f4697b07ce815f004ac9ea14bb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a822f4697b07ce815f004ac9ea14bb2 columnFamilyName B 2024-11-21T03:40:31,551 DEBUG [StoreOpener-9a822f4697b07ce815f004ac9ea14bb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:31,551 INFO [StoreOpener-9a822f4697b07ce815f004ac9ea14bb2-1 {}] regionserver.HStore(327): Store=9a822f4697b07ce815f004ac9ea14bb2/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:40:31,552 INFO [StoreOpener-9a822f4697b07ce815f004ac9ea14bb2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:31,552 INFO [StoreOpener-9a822f4697b07ce815f004ac9ea14bb2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a822f4697b07ce815f004ac9ea14bb2 columnFamilyName C 2024-11-21T03:40:31,552 DEBUG [StoreOpener-9a822f4697b07ce815f004ac9ea14bb2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:31,553 INFO [StoreOpener-9a822f4697b07ce815f004ac9ea14bb2-1 {}] regionserver.HStore(327): Store=9a822f4697b07ce815f004ac9ea14bb2/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:40:31,553 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:31,553 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:31,554 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:31,555 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:31,555 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:31,556 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T03:40:31,557 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:31,558 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 9a822f4697b07ce815f004ac9ea14bb2; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65413430, jitterRate=-0.025263935327529907}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T03:40:31,558 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:31,559 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 9a822f4697b07ce815f004ac9ea14bb2: Running coprocessor pre-open hook at 1732160431535Writing region info on filesystem at 1732160431535Initializing all the Stores at 1732160431536 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732160431536Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732160431537 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732160431537Cleaning up temporary data from old regions at 1732160431555 (+18 ms)Running coprocessor post-open hooks at 1732160431558 (+3 ms)Region opened successfully at 1732160431559 (+1 ms) 2024-11-21T03:40:31,560 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., pid=47, masterSystemTime=1732160431530 2024-11-21T03:40:31,562 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:31,562 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:31,562 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=45 updating hbase:meta row=9a822f4697b07ce815f004ac9ea14bb2, regionState=OPEN, openSeqNum=5, regionLocation=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:31,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=45, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 because future has completed 2024-11-21T03:40:31,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=45 2024-11-21T03:40:31,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=45, state=SUCCESS, hasLock=false; OpenRegionProcedure 9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 in 187 msec 2024-11-21T03:40:31,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-21T03:40:31,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a822f4697b07ce815f004ac9ea14bb2, REOPEN/MOVE in 511 msec 2024-11-21T03:40:31,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=43 2024-11-21T03:40:31,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=43, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 522 msec 2024-11-21T03:40:31,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 955 msec 2024-11-21T03:40:31,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=43 2024-11-21T03:40:31,575 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:31,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ed5d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,576 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:31,576 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:31,577 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:31,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:31,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:31,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f4dcb24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:31,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:31,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:31,578 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52e204e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4406a4b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,580 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:31,580 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:31,581 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:31,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:31,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:31,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e2769e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:31,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:31,581 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:31,582 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40340, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:31,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d06fe16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,584 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d0a90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,584 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:31,584 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:31,585 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a585600, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:31,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:31,586 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40344, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:31,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60cd8090, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b236c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:31,588 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:31,589 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@263eaf48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:31,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:31,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:31,591 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40360, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:31,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8ebc16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26dcfede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,592 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:31,592 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:31,593 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:31,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:31,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:31,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f3563ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:31,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:31,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:31,595 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:31,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@666b1882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ae3464, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,596 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:31,596 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:31,597 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:31,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:31,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@589a8239, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:31,599 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b576a30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bbb11d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,600 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:31,600 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:31,601 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:31,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:31,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25baa411, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:31,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:31,603 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40418, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:31,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a3eeec9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d2a6fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,604 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:31,604 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:31,605 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:31,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:31,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:31,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b5b20b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:31,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:31,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:31,606 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40422, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:31,607 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5784fa65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30ade04f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,608 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:31,608 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:31,608 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:31,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:31,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:31,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fde552a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:31,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:31,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:31,610 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40426, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:31,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48881205, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:31,611 DEBUG [Thread-665 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:31,611 DEBUG [Thread-676 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:31,611 DEBUG [Thread-663 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:31,611 DEBUG [Thread-674 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:31,611 DEBUG [Thread-659 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:31,611 DEBUG [Thread-670 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:31,612 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:40:31,612 DEBUG [Thread-672 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:31,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:31,612 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:31,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:31,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:31,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:31,612 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:31,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:31,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:31,613 DEBUG [Thread-661 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:31,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:31,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:31,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:31,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:31,614 DEBUG [Thread-667 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:31,614 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37514, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:31,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:31,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:31,614 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37512, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:31,614 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:31,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:31,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:31,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:31,615 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:31,615 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:31,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-21T03:40:31,615 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:31,615 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37578, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:31,615 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T03:40:31,616 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:31,616 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37592, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:31,617 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:40:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5] 2024-11-21T03:40:31,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5] 2024-11-21T03:40:31,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5] 2024-11-21T03:40:31,619 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:40:31,619 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:40:31,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5] 2024-11-21T03:40:31,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5] 2024-11-21T03:40:31,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5] 2024-11-21T03:40:31,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5] 2024-11-21T03:40:31,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5] 2024-11-21T03:40:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5] 2024-11-21T03:40:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:31,634 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:31,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a822f4697b07ce815f004ac9ea14bb2 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:40:31,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121406b4b62af2c4f479a0f69218757dc1b_9a822f4697b07ce815f004ac9ea14bb2 is 50, key is test_row_0/A:col10/1732160431630/Put/seqid=0 2024-11-21T03:40:31,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160491676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:31,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:31,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:31,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:31,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160491676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:31,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:31,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:31,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:31,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741957_1133 (size=14594) 2024-11-21T03:40:31,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:31,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160491678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:31,691 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:31,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:31,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:31,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:31,691 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:31,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160491680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:31,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:31,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:31,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:31,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:31,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160491681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:31,693 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:31,693 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:31,693 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:31,697 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121406b4b62af2c4f479a0f69218757dc1b_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121406b4b62af2c4f479a0f69218757dc1b_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:31,699 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/7680e8e727104082b4d12bd395b20f42, store: [table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:31,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/7680e8e727104082b4d12bd395b20f42 is 175, key is test_row_0/A:col10/1732160431630/Put/seqid=0 2024-11-21T03:40:31,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741958_1134 (size=39549) 2024-11-21T03:40:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T03:40:31,732 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/7680e8e727104082b4d12bd395b20f42 2024-11-21T03:40:31,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/43dffd27eda04baf8b20f32c377438fc is 50, key is test_row_0/B:col10/1732160431630/Put/seqid=0 2024-11-21T03:40:31,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T03:40:31,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:31,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:31,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:31,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:31,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:31,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:31,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741959_1135 (size=12001) 2024-11-21T03:40:31,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:31,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160491798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:31,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:31,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160491798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:31,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:31,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:31,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160491798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:31,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:31,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:31,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:31,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:31,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160491798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:31,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:31,800 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:31,801 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:31,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:31,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160491799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:31,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T03:40:31,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:31,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:31,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:31,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:31,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:31,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:31,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T03:40:32,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:32,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160492007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:32,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:32,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:32,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160492007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:32,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:32,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:32,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160492007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:32,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:32,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:32,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160492007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:32,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:32,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:32,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160492007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:32,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:32,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T03:40:32,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:32,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:32,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:32,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:32,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:32,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:32,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/43dffd27eda04baf8b20f32c377438fc 2024-11-21T03:40:32,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/3d4616d27e974165913fa0a397433395 is 50, key is test_row_0/C:col10/1732160431630/Put/seqid=0 2024-11-21T03:40:32,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T03:40:32,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:32,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:32,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:32,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:32,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:32,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:32,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T03:40:32,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741960_1136 (size=12001) 2024-11-21T03:40:32,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/3d4616d27e974165913fa0a397433395 2024-11-21T03:40:32,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/7680e8e727104082b4d12bd395b20f42 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/7680e8e727104082b4d12bd395b20f42 2024-11-21T03:40:32,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/7680e8e727104082b4d12bd395b20f42, entries=200, sequenceid=15, filesize=38.6 K 2024-11-21T03:40:32,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/43dffd27eda04baf8b20f32c377438fc as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/43dffd27eda04baf8b20f32c377438fc 2024-11-21T03:40:32,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:32,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160492317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:32,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:32,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:32,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160492317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:32,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:32,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160492318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:32,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:32,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:32,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:32,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160492318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:32,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:32,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/43dffd27eda04baf8b20f32c377438fc, entries=150, sequenceid=15, filesize=11.7 K 2024-11-21T03:40:32,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/3d4616d27e974165913fa0a397433395 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/3d4616d27e974165913fa0a397433395 2024-11-21T03:40:32,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/3d4616d27e974165913fa0a397433395, entries=150, sequenceid=15, filesize=11.7 K 2024-11-21T03:40:32,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 9a822f4697b07ce815f004ac9ea14bb2 in 709ms, sequenceid=15, compaction requested=false 2024-11-21T03:40:32,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:32,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:32,347 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:32,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a822f4697b07ce815f004ac9ea14bb2 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T03:40:32,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216980f433a0724ce18868d4b3e3dbad1b_9a822f4697b07ce815f004ac9ea14bb2 is 50, key is test_row_0/A:col10/1732160431671/Put/seqid=0 2024-11-21T03:40:32,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741961_1137 (size=14594) 2024-11-21T03:40:32,389 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:32,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T03:40:32,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:32,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:32,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:32,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:32,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:32,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:32,402 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216980f433a0724ce18868d4b3e3dbad1b_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216980f433a0724ce18868d4b3e3dbad1b_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:32,403 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/cef507059fa84b49a112fb85428f140b, store: [table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:32,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/cef507059fa84b49a112fb85428f140b is 175, key is test_row_0/A:col10/1732160431671/Put/seqid=0 2024-11-21T03:40:32,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:32,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160492428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:32,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:32,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741962_1138 (size=39549) 2024-11-21T03:40:32,455 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/cef507059fa84b49a112fb85428f140b 2024-11-21T03:40:32,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/a19061041a9a46a9abec7d873814b540 is 50, key is test_row_0/B:col10/1732160431671/Put/seqid=0 2024-11-21T03:40:32,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741963_1139 (size=12001) 2024-11-21T03:40:32,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/a19061041a9a46a9abec7d873814b540 2024-11-21T03:40:32,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/c8c2a8d1760b4e86866c9a0207e03768 is 50, key is test_row_0/C:col10/1732160431671/Put/seqid=0 2024-11-21T03:40:32,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T03:40:32,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:32,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:32,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:32,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:32,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:32,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:32,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:32,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160492556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:32,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:32,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741964_1140 (size=12001) 2024-11-21T03:40:32,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/c8c2a8d1760b4e86866c9a0207e03768 2024-11-21T03:40:32,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/cef507059fa84b49a112fb85428f140b as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/cef507059fa84b49a112fb85428f140b 2024-11-21T03:40:32,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/cef507059fa84b49a112fb85428f140b, entries=200, sequenceid=41, filesize=38.6 K 2024-11-21T03:40:32,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/a19061041a9a46a9abec7d873814b540 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/a19061041a9a46a9abec7d873814b540 2024-11-21T03:40:32,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/a19061041a9a46a9abec7d873814b540, entries=150, sequenceid=41, filesize=11.7 K 2024-11-21T03:40:32,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/c8c2a8d1760b4e86866c9a0207e03768 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/c8c2a8d1760b4e86866c9a0207e03768 2024-11-21T03:40:32,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/c8c2a8d1760b4e86866c9a0207e03768, entries=150, sequenceid=41, filesize=11.7 K 2024-11-21T03:40:32,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 9a822f4697b07ce815f004ac9ea14bb2 in 271ms, sequenceid=41, compaction requested=false 2024-11-21T03:40:32,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:32,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T03:40:32,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:32,705 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 9a822f4697b07ce815f004ac9ea14bb2 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-21T03:40:32,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a229e77cad22470cb74c44c0ea16e430_9a822f4697b07ce815f004ac9ea14bb2 is 50, key is test_row_0/A:col10/1732160432386/Put/seqid=0 2024-11-21T03:40:32,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741965_1141 (size=12154) 2024-11-21T03:40:32,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:32,748 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a229e77cad22470cb74c44c0ea16e430_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a229e77cad22470cb74c44c0ea16e430_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:32,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/64b136fa72c94b2a8c6afe8cae971488, store: [table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:32,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/64b136fa72c94b2a8c6afe8cae971488 is 175, key is test_row_0/A:col10/1732160432386/Put/seqid=0 2024-11-21T03:40:32,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T03:40:32,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741966_1142 (size=30955) 2024-11-21T03:40:32,767 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/64b136fa72c94b2a8c6afe8cae971488 2024-11-21T03:40:32,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:32,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:32,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/a0fd47f2ad384a7d8a2061aac5e9c3b7 is 50, key is test_row_0/B:col10/1732160432386/Put/seqid=0 2024-11-21T03:40:32,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741967_1143 (size=12001) 2024-11-21T03:40:32,839 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/a0fd47f2ad384a7d8a2061aac5e9c3b7 2024-11-21T03:40:32,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/0878c8009a594825b1649e61478b1d25 is 50, key is test_row_0/C:col10/1732160432386/Put/seqid=0 2024-11-21T03:40:32,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741968_1144 (size=12001) 2024-11-21T03:40:32,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:32,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160492879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:32,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:32,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:32,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160492881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:32,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:32,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:32,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160492884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:32,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:32,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160492886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:32,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:32,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:32,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160492889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:33,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:33,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160492997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:33,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:33,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160492997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:33,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160492997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,001 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:33,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:33,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160493007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:33,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:33,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160493007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:33,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:33,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:33,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:33,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160493207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:33,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160493208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:33,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160493208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:33,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:33,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:33,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:33,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160493217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:33,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:33,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160493217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:33,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:33,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:33,273 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/0878c8009a594825b1649e61478b1d25 2024-11-21T03:40:33,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/64b136fa72c94b2a8c6afe8cae971488 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/64b136fa72c94b2a8c6afe8cae971488 2024-11-21T03:40:33,291 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/64b136fa72c94b2a8c6afe8cae971488, entries=150, sequenceid=51, filesize=30.2 K 2024-11-21T03:40:33,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/a0fd47f2ad384a7d8a2061aac5e9c3b7 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/a0fd47f2ad384a7d8a2061aac5e9c3b7 2024-11-21T03:40:33,301 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/a0fd47f2ad384a7d8a2061aac5e9c3b7, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T03:40:33,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/0878c8009a594825b1649e61478b1d25 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/0878c8009a594825b1649e61478b1d25 2024-11-21T03:40:33,334 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/0878c8009a594825b1649e61478b1d25, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T03:40:33,335 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 9a822f4697b07ce815f004ac9ea14bb2 in 630ms, sequenceid=51, compaction requested=true 2024-11-21T03:40:33,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:33,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:33,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-21T03:40:33,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-21T03:40:33,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-21T03:40:33,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7190 sec 2024-11-21T03:40:33,343 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.7280 sec 2024-11-21T03:40:33,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:33,531 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:33,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a822f4697b07ce815f004ac9ea14bb2 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-21T03:40:33,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:33,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160493547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:33,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:33,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160493547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:33,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160493549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:33,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160493551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:33,559 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160493553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:33,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121695bfaa2ccfb43df924712df1f35cdfb_9a822f4697b07ce815f004ac9ea14bb2 is 50, key is test_row_0/A:col10/1732160433522/Put/seqid=0 2024-11-21T03:40:33,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741969_1145 (size=17034) 2024-11-21T03:40:33,587 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:33,599 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121695bfaa2ccfb43df924712df1f35cdfb_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121695bfaa2ccfb43df924712df1f35cdfb_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:33,606 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/0f017242894b4926a6e040432fd88d17, store: [table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:33,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/0f017242894b4926a6e040432fd88d17 is 175, key is test_row_0/A:col10/1732160433522/Put/seqid=0 2024-11-21T03:40:33,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741970_1146 (size=48139) 2024-11-21T03:40:33,619 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/0f017242894b4926a6e040432fd88d17 2024-11-21T03:40:33,621 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T03:40:33,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/a8f078debfe74eeb9549f8ac4f882428 is 50, key is test_row_0/B:col10/1732160433522/Put/seqid=0 2024-11-21T03:40:33,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160493667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:33,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:33,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160493667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:33,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160493667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:33,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:33,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:33,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:33,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160493668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:33,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:33,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160493668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:33,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:33,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741971_1147 (size=12001) 2024-11-21T03:40:33,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T03:40:33,767 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:33,768 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:40:33,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-21T03:40:33,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T03:40:33,771 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:40:33,772 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:40:33,772 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:40:33,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T03:40:33,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:33,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160493877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:33,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:33,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:33,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160493887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:33,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:33,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160493887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:33,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,888 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:33,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:33,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160493887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:33,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160493887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:33,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T03:40:33,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:33,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:33,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:33,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:33,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:34,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T03:40:34,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:34,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:34,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:34,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:34,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:34,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:34,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T03:40:34,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/a8f078debfe74eeb9549f8ac4f882428 2024-11-21T03:40:34,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/4438bd6e956f4eb090f23447fd267968 is 50, key is test_row_0/C:col10/1732160433522/Put/seqid=0 2024-11-21T03:40:34,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741972_1148 (size=12001) 2024-11-21T03:40:34,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/4438bd6e956f4eb090f23447fd267968 2024-11-21T03:40:34,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/0f017242894b4926a6e040432fd88d17 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/0f017242894b4926a6e040432fd88d17 2024-11-21T03:40:34,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/0f017242894b4926a6e040432fd88d17, entries=250, sequenceid=80, filesize=47.0 K 2024-11-21T03:40:34,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:34,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160494188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:34,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:34,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:34,190 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:34,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/a8f078debfe74eeb9549f8ac4f882428 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/a8f078debfe74eeb9549f8ac4f882428 2024-11-21T03:40:34,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/a8f078debfe74eeb9549f8ac4f882428, entries=150, sequenceid=80, filesize=11.7 K 2024-11-21T03:40:34,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:34,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160494197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:34,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:34,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160494197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:34,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:34,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:34,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:34,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:34,203 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:34,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:34,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:34,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160494197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:34,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:34,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:34,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:34,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:34,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160494197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:34,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/4438bd6e956f4eb090f23447fd267968 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/4438bd6e956f4eb090f23447fd267968 2024-11-21T03:40:34,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:34,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:34,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:34,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/4438bd6e956f4eb090f23447fd267968, entries=150, sequenceid=80, filesize=11.7 K 2024-11-21T03:40:34,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=33.54 KB/34350 for 9a822f4697b07ce815f004ac9ea14bb2 in 680ms, sequenceid=80, compaction requested=true 2024-11-21T03:40:34,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:34,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a822f4697b07ce815f004ac9ea14bb2:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:40:34,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:34,212 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:40:34,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a822f4697b07ce815f004ac9ea14bb2:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:40:34,212 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:40:34,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:34,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a822f4697b07ce815f004ac9ea14bb2:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:40:34,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:34,214 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 158192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:40:34,214 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 9a822f4697b07ce815f004ac9ea14bb2/A is initiating minor compaction (all files) 2024-11-21T03:40:34,214 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:40:34,214 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a822f4697b07ce815f004ac9ea14bb2/A in TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:34,214 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 9a822f4697b07ce815f004ac9ea14bb2/B is initiating minor compaction (all files) 2024-11-21T03:40:34,214 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/7680e8e727104082b4d12bd395b20f42, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/cef507059fa84b49a112fb85428f140b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/64b136fa72c94b2a8c6afe8cae971488, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/0f017242894b4926a6e040432fd88d17] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp, totalSize=154.5 K 2024-11-21T03:40:34,214 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a822f4697b07ce815f004ac9ea14bb2/B in TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:34,214 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/43dffd27eda04baf8b20f32c377438fc, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/a19061041a9a46a9abec7d873814b540, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/a0fd47f2ad384a7d8a2061aac5e9c3b7, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/a8f078debfe74eeb9549f8ac4f882428] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp, totalSize=46.9 K 2024-11-21T03:40:34,214 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:34,215 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. files: [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/7680e8e727104082b4d12bd395b20f42, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/cef507059fa84b49a112fb85428f140b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/64b136fa72c94b2a8c6afe8cae971488, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/0f017242894b4926a6e040432fd88d17] 2024-11-21T03:40:34,216 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7680e8e727104082b4d12bd395b20f42, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732160431624 2024-11-21T03:40:34,216 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 43dffd27eda04baf8b20f32c377438fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732160431629 2024-11-21T03:40:34,219 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting a19061041a9a46a9abec7d873814b540, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732160431671 2024-11-21T03:40:34,219 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting cef507059fa84b49a112fb85428f140b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732160431671 2024-11-21T03:40:34,220 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting a0fd47f2ad384a7d8a2061aac5e9c3b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732160432386 2024-11-21T03:40:34,221 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64b136fa72c94b2a8c6afe8cae971488, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732160432386 2024-11-21T03:40:34,221 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting a8f078debfe74eeb9549f8ac4f882428, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732160432881 2024-11-21T03:40:34,224 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f017242894b4926a6e040432fd88d17, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732160432881 2024-11-21T03:40:34,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T03:40:34,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:34,235 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 9a822f4697b07ce815f004ac9ea14bb2 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-21T03:40:34,250 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a822f4697b07ce815f004ac9ea14bb2#B#compaction#129 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:34,255 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/83dfbaa3d10c447ca9400d38aedaae7b is 50, key is test_row_0/B:col10/1732160433522/Put/seqid=0 2024-11-21T03:40:34,267 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:34,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112177996bcea5c94d4f9a091659c8e6150a_9a822f4697b07ce815f004ac9ea14bb2 is 50, key is test_row_0/A:col10/1732160433542/Put/seqid=0 2024-11-21T03:40:34,300 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112124f6ea8a04f349459a7058f09c9bbc1e_9a822f4697b07ce815f004ac9ea14bb2 store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:34,307 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112124f6ea8a04f349459a7058f09c9bbc1e_9a822f4697b07ce815f004ac9ea14bb2, store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:34,307 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112124f6ea8a04f349459a7058f09c9bbc1e_9a822f4697b07ce815f004ac9ea14bb2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:34,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741973_1149 (size=12139) 2024-11-21T03:40:34,326 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/83dfbaa3d10c447ca9400d38aedaae7b as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/83dfbaa3d10c447ca9400d38aedaae7b 2024-11-21T03:40:34,339 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9a822f4697b07ce815f004ac9ea14bb2/B of 9a822f4697b07ce815f004ac9ea14bb2 into 83dfbaa3d10c447ca9400d38aedaae7b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:34,339 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:34,339 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., storeName=9a822f4697b07ce815f004ac9ea14bb2/B, priority=12, startTime=1732160434212; duration=0sec 2024-11-21T03:40:34,339 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:34,339 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a822f4697b07ce815f004ac9ea14bb2:B 2024-11-21T03:40:34,340 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:40:34,343 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:40:34,344 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 9a822f4697b07ce815f004ac9ea14bb2/C is initiating minor compaction (all files) 2024-11-21T03:40:34,344 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a822f4697b07ce815f004ac9ea14bb2/C in TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:34,344 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/3d4616d27e974165913fa0a397433395, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/c8c2a8d1760b4e86866c9a0207e03768, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/0878c8009a594825b1649e61478b1d25, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/4438bd6e956f4eb090f23447fd267968] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp, totalSize=46.9 K 2024-11-21T03:40:34,345 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d4616d27e974165913fa0a397433395, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732160431629 2024-11-21T03:40:34,345 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting c8c2a8d1760b4e86866c9a0207e03768, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732160431671 2024-11-21T03:40:34,346 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0878c8009a594825b1649e61478b1d25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732160432386 2024-11-21T03:40:34,346 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 4438bd6e956f4eb090f23447fd267968, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732160432881 2024-11-21T03:40:34,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741974_1150 (size=12154) 2024-11-21T03:40:34,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,375 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112177996bcea5c94d4f9a091659c8e6150a_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112177996bcea5c94d4f9a091659c8e6150a_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:34,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/87ca946911b748afbafcfce3f47f48e3, store: [table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:34,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/87ca946911b748afbafcfce3f47f48e3 is 175, key is test_row_0/A:col10/1732160433542/Put/seqid=0 2024-11-21T03:40:34,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741975_1151 (size=4469) 2024-11-21T03:40:34,390 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a822f4697b07ce815f004ac9ea14bb2#C#compaction#132 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:34,391 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/1fb3e881904f44288da74d02cb746fb9 is 50, key is test_row_0/C:col10/1732160433522/Put/seqid=0 2024-11-21T03:40:34,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T03:40:34,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741976_1152 (size=30955) 2024-11-21T03:40:34,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741977_1153 (size=12139) 2024-11-21T03:40:34,425 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=88, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/87ca946911b748afbafcfce3f47f48e3 2024-11-21T03:40:34,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/9edddfaeee374d2689ca2596fb74495f is 50, key is test_row_0/B:col10/1732160433542/Put/seqid=0 2024-11-21T03:40:34,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741978_1154 (size=12001) 2024-11-21T03:40:34,455 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/9edddfaeee374d2689ca2596fb74495f 2024-11-21T03:40:34,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/6d5d3a99e71e46c3b2e3eeeb5401149f is 50, key is test_row_0/C:col10/1732160433542/Put/seqid=0 2024-11-21T03:40:34,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741979_1155 (size=12001) 2024-11-21T03:40:34,499 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/6d5d3a99e71e46c3b2e3eeeb5401149f 2024-11-21T03:40:34,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/87ca946911b748afbafcfce3f47f48e3 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/87ca946911b748afbafcfce3f47f48e3 2024-11-21T03:40:34,524 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/87ca946911b748afbafcfce3f47f48e3, entries=150, sequenceid=88, filesize=30.2 K 2024-11-21T03:40:34,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/9edddfaeee374d2689ca2596fb74495f as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/9edddfaeee374d2689ca2596fb74495f 2024-11-21T03:40:34,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,533 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/9edddfaeee374d2689ca2596fb74495f, entries=150, sequenceid=88, filesize=11.7 K 2024-11-21T03:40:34,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/6d5d3a99e71e46c3b2e3eeeb5401149f as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/6d5d3a99e71e46c3b2e3eeeb5401149f 2024-11-21T03:40:34,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,547 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/6d5d3a99e71e46c3b2e3eeeb5401149f, entries=150, sequenceid=88, filesize=11.7 K 2024-11-21T03:40:34,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,548 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 9a822f4697b07ce815f004ac9ea14bb2 in 313ms, sequenceid=88, compaction requested=false 2024-11-21T03:40:34,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:34,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:34,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-21T03:40:34,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-21T03:40:34,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,555 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-21T03:40:34,555 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 779 msec 2024-11-21T03:40:34,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,559 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 788 msec 2024-11-21T03:40:34,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:34,768 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:34,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a822f4697b07ce815f004ac9ea14bb2 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:40:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,782 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a822f4697b07ce815f004ac9ea14bb2#A#compaction#130 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:34,783 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/0add856224c14382844ea10e8a3945d0 is 175, key is test_row_0/A:col10/1732160433522/Put/seqid=0 2024-11-21T03:40:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741980_1156 (size=31093) 2024-11-21T03:40:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121986cc79522dc44caaa7c8c7b1a5d7f6c_9a822f4697b07ce815f004ac9ea14bb2 is 50, key is test_row_0/A:col10/1732160434766/Put/seqid=0 2024-11-21T03:40:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,811 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/0add856224c14382844ea10e8a3945d0 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/0add856224c14382844ea10e8a3945d0 2024-11-21T03:40:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,821 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9a822f4697b07ce815f004ac9ea14bb2/A of 9a822f4697b07ce815f004ac9ea14bb2 into 0add856224c14382844ea10e8a3945d0(size=30.4 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:34,821 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:34,821 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., storeName=9a822f4697b07ce815f004ac9ea14bb2/A, priority=12, startTime=1732160434212; duration=0sec 2024-11-21T03:40:34,821 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:34,821 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a822f4697b07ce815f004ac9ea14bb2:A 2024-11-21T03:40:34,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160494819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:34,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:34,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:34,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:34,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741982_1158 (size=26798) 2024-11-21T03:40:34,833 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:34,874 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/1fb3e881904f44288da74d02cb746fb9 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/1fb3e881904f44288da74d02cb746fb9 2024-11-21T03:40:34,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160494828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:34,881 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121986cc79522dc44caaa7c8c7b1a5d7f6c_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121986cc79522dc44caaa7c8c7b1a5d7f6c_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:34,884 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/b01c10ac959e4413bad147aa5a293662, store: [table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:34,884 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9a822f4697b07ce815f004ac9ea14bb2/C of 9a822f4697b07ce815f004ac9ea14bb2 into 1fb3e881904f44288da74d02cb746fb9(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:34,884 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:34,884 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., storeName=9a822f4697b07ce815f004ac9ea14bb2/C, priority=12, startTime=1732160434212; duration=0sec 2024-11-21T03:40:34,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/b01c10ac959e4413bad147aa5a293662 is 175, key is test_row_0/A:col10/1732160434766/Put/seqid=0 2024-11-21T03:40:34,884 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:34,884 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a822f4697b07ce815f004ac9ea14bb2:C 2024-11-21T03:40:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:34,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160494828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:34,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:34,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:34,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:34,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160494830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:34,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741981_1157 (size=82585) 2024-11-21T03:40:34,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160494830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:34,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:34,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:34,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:34,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:34,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:34,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:34,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T03:40:34,907 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:34,908 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:40:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-21T03:40:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T03:40:34,911 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:40:34,912 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:40:34,912 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:40:34,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160494938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:34,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:34,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:34,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:35,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160494997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:35,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160494997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:35,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:35,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160494997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:35,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:35,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160494997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:35,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:35,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T03:40:35,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T03:40:35,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:35,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:35,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:35,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:35,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:35,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:35,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:35,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160495148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:35,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160495207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:35,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160495207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:35,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160495207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:35,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:35,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160495207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:35,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:35,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:35,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T03:40:35,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:35,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:35,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:35,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:35,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:35,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:35,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T03:40:35,288 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=101, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/b01c10ac959e4413bad147aa5a293662 2024-11-21T03:40:35,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/c7e68081a0b6420f9c44f824ad37f7b3 is 50, key is test_row_0/B:col10/1732160434766/Put/seqid=0 2024-11-21T03:40:35,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741983_1159 (size=12001) 2024-11-21T03:40:35,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/c7e68081a0b6420f9c44f824ad37f7b3 2024-11-21T03:40:35,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/17ec14b7211e40eca3f21cd24ac33c2b is 50, key is test_row_0/C:col10/1732160434766/Put/seqid=0 2024-11-21T03:40:35,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741984_1160 (size=12001) 2024-11-21T03:40:35,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=101 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/17ec14b7211e40eca3f21cd24ac33c2b 2024-11-21T03:40:35,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T03:40:35,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:35,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:35,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:35,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:35,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:35,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:35,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/b01c10ac959e4413bad147aa5a293662 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/b01c10ac959e4413bad147aa5a293662 2024-11-21T03:40:35,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/b01c10ac959e4413bad147aa5a293662, entries=450, sequenceid=101, filesize=80.6 K 2024-11-21T03:40:35,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/c7e68081a0b6420f9c44f824ad37f7b3 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/c7e68081a0b6420f9c44f824ad37f7b3 2024-11-21T03:40:35,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/c7e68081a0b6420f9c44f824ad37f7b3, entries=150, sequenceid=101, filesize=11.7 K 2024-11-21T03:40:35,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/17ec14b7211e40eca3f21cd24ac33c2b as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/17ec14b7211e40eca3f21cd24ac33c2b 2024-11-21T03:40:35,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/17ec14b7211e40eca3f21cd24ac33c2b, entries=150, sequenceid=101, filesize=11.7 K 2024-11-21T03:40:35,419 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-21T03:40:35,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 9a822f4697b07ce815f004ac9ea14bb2 in 653ms, sequenceid=101, compaction requested=true 2024-11-21T03:40:35,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:35,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a822f4697b07ce815f004ac9ea14bb2:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:40:35,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:35,422 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:35,422 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:35,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a822f4697b07ce815f004ac9ea14bb2:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:40:35,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:35,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a822f4697b07ce815f004ac9ea14bb2:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:40:35,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:35,423 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:35,423 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 9a822f4697b07ce815f004ac9ea14bb2/B is initiating minor compaction (all files) 2024-11-21T03:40:35,423 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a822f4697b07ce815f004ac9ea14bb2/B in TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:35,423 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/83dfbaa3d10c447ca9400d38aedaae7b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/9edddfaeee374d2689ca2596fb74495f, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/c7e68081a0b6420f9c44f824ad37f7b3] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp, totalSize=35.3 K 2024-11-21T03:40:35,424 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 83dfbaa3d10c447ca9400d38aedaae7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732160432881 2024-11-21T03:40:35,428 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 144633 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:35,428 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 9a822f4697b07ce815f004ac9ea14bb2/A is initiating minor compaction (all files) 2024-11-21T03:40:35,428 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a822f4697b07ce815f004ac9ea14bb2/A in TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:35,428 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/0add856224c14382844ea10e8a3945d0, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/87ca946911b748afbafcfce3f47f48e3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/b01c10ac959e4413bad147aa5a293662] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp, totalSize=141.2 K 2024-11-21T03:40:35,428 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:35,428 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. files: [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/0add856224c14382844ea10e8a3945d0, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/87ca946911b748afbafcfce3f47f48e3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/b01c10ac959e4413bad147aa5a293662] 2024-11-21T03:40:35,429 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 9edddfaeee374d2689ca2596fb74495f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732160433542 2024-11-21T03:40:35,429 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting c7e68081a0b6420f9c44f824ad37f7b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732160434755 2024-11-21T03:40:35,429 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0add856224c14382844ea10e8a3945d0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732160432881 2024-11-21T03:40:35,430 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87ca946911b748afbafcfce3f47f48e3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732160433542 2024-11-21T03:40:35,433 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting b01c10ac959e4413bad147aa5a293662, keycount=450, bloomtype=ROW, size=80.6 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732160434749 2024-11-21T03:40:35,452 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a822f4697b07ce815f004ac9ea14bb2#B#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:35,453 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/a0fb333243a24192b010c3ef4843a657 is 50, key is test_row_0/B:col10/1732160434766/Put/seqid=0 2024-11-21T03:40:35,460 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:35,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:35,461 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:35,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a822f4697b07ce815f004ac9ea14bb2 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T03:40:35,479 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112142bcb63d39414de692b26e0607341f97_9a822f4697b07ce815f004ac9ea14bb2 store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:35,484 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112142bcb63d39414de692b26e0607341f97_9a822f4697b07ce815f004ac9ea14bb2, store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:35,484 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112142bcb63d39414de692b26e0607341f97_9a822f4697b07ce815f004ac9ea14bb2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:35,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a7d2990f32a24106a0df459801c1847c_9a822f4697b07ce815f004ac9ea14bb2 is 50, key is test_row_0/A:col10/1732160434823/Put/seqid=0 2024-11-21T03:40:35,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741985_1161 (size=12241) 2024-11-21T03:40:35,506 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/a0fb333243a24192b010c3ef4843a657 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/a0fb333243a24192b010c3ef4843a657 2024-11-21T03:40:35,515 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a822f4697b07ce815f004ac9ea14bb2/B of 9a822f4697b07ce815f004ac9ea14bb2 into a0fb333243a24192b010c3ef4843a657(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:35,515 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:35,515 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., storeName=9a822f4697b07ce815f004ac9ea14bb2/B, priority=13, startTime=1732160435422; duration=0sec 2024-11-21T03:40:35,515 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:35,515 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a822f4697b07ce815f004ac9ea14bb2:B 2024-11-21T03:40:35,515 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:35,517 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:35,517 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 9a822f4697b07ce815f004ac9ea14bb2/C is initiating minor compaction (all files) 2024-11-21T03:40:35,517 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a822f4697b07ce815f004ac9ea14bb2/C in TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:35,517 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/1fb3e881904f44288da74d02cb746fb9, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/6d5d3a99e71e46c3b2e3eeeb5401149f, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/17ec14b7211e40eca3f21cd24ac33c2b] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp, totalSize=35.3 K 2024-11-21T03:40:35,518 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fb3e881904f44288da74d02cb746fb9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732160432881 2024-11-21T03:40:35,520 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d5d3a99e71e46c3b2e3eeeb5401149f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732160433542 2024-11-21T03:40:35,520 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 17ec14b7211e40eca3f21cd24ac33c2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732160434755 2024-11-21T03:40:35,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160495517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:35,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160495517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:35,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:35,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160495517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:35,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741986_1162 (size=4469) 2024-11-21T03:40:35,525 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a822f4697b07ce815f004ac9ea14bb2#A#compaction#139 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:35,525 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/f66f90cdc25c40d4a017e7a1b3178fb5 is 175, key is test_row_0/A:col10/1732160434766/Put/seqid=0 2024-11-21T03:40:35,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T03:40:35,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:35,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:35,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:35,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:35,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:35,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:35,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:35,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160495519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:35,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:35,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160495521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:35,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:35,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T03:40:35,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741987_1163 (size=14594) 2024-11-21T03:40:35,554 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a822f4697b07ce815f004ac9ea14bb2#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:35,555 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/f7b533b213884b1f84e3c794495d162a is 50, key is test_row_0/C:col10/1732160434766/Put/seqid=0 2024-11-21T03:40:35,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741988_1164 (size=31195) 2024-11-21T03:40:35,579 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/f66f90cdc25c40d4a017e7a1b3178fb5 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/f66f90cdc25c40d4a017e7a1b3178fb5 2024-11-21T03:40:35,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741989_1165 (size=12241) 2024-11-21T03:40:35,594 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a822f4697b07ce815f004ac9ea14bb2/A of 9a822f4697b07ce815f004ac9ea14bb2 into f66f90cdc25c40d4a017e7a1b3178fb5(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:35,594 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:35,594 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., storeName=9a822f4697b07ce815f004ac9ea14bb2/A, priority=13, startTime=1732160435421; duration=0sec 2024-11-21T03:40:35,594 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:35,594 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a822f4697b07ce815f004ac9ea14bb2:A 2024-11-21T03:40:35,599 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/f7b533b213884b1f84e3c794495d162a as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/f7b533b213884b1f84e3c794495d162a 2024-11-21T03:40:35,613 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a822f4697b07ce815f004ac9ea14bb2/C of 9a822f4697b07ce815f004ac9ea14bb2 into f7b533b213884b1f84e3c794495d162a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:35,613 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:35,613 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., storeName=9a822f4697b07ce815f004ac9ea14bb2/C, priority=13, startTime=1732160435422; duration=0sec 2024-11-21T03:40:35,613 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:35,613 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a822f4697b07ce815f004ac9ea14bb2:C 2024-11-21T03:40:35,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160495637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:35,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:35,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160495637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:35,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:35,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T03:40:35,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:35,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:35,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:35,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:35,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:35,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:35,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T03:40:35,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:35,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:35,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:35,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:35,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:35,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:35,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160495847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:35,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:35,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160495848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:35,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:35,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:35,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:35,944 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:35,953 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a7d2990f32a24106a0df459801c1847c_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a7d2990f32a24106a0df459801c1847c_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:35,957 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/305ccb6b5e114c21a96b29e15013416b, store: [table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:35,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/305ccb6b5e114c21a96b29e15013416b is 175, key is test_row_0/A:col10/1732160434823/Put/seqid=0 2024-11-21T03:40:35,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T03:40:36,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:36,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:36,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:36,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:36,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:36,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741990_1166 (size=39549) 2024-11-21T03:40:36,026 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=128, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/305ccb6b5e114c21a96b29e15013416b 2024-11-21T03:40:36,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160496027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:36,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:36,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160496027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:36,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:36,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:36,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:36,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:36,034 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:36,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:36,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160496037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:36,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:36,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T03:40:36,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/81f6c39659d44b21b852dabd3fe61367 is 50, key is test_row_0/B:col10/1732160434823/Put/seqid=0 2024-11-21T03:40:36,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741991_1167 (size=12001) 2024-11-21T03:40:36,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T03:40:36,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:36,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:36,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:36,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:36,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:36,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:36,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160496157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:36,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:36,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160496157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:36,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:36,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T03:40:36,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:36,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:36,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:36,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:36,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:36,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:36,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T03:40:36,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:36,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:36,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:36,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:36,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:36,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/81f6c39659d44b21b852dabd3fe61367 2024-11-21T03:40:36,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/ee8e3b9f7f35461e9dcd9320c4dce2fe is 50, key is test_row_0/C:col10/1732160434823/Put/seqid=0 2024-11-21T03:40:36,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741992_1168 (size=12001) 2024-11-21T03:40:36,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T03:40:36,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:36,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:36,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:36,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:36,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:36,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160496667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:36,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:36,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:36,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:36,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160496668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:36,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:36,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T03:40:36,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:36,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:36,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:36,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:36,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:36,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/ee8e3b9f7f35461e9dcd9320c4dce2fe 2024-11-21T03:40:36,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T03:40:36,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:36,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:36,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:36,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:36,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:36,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:36,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/305ccb6b5e114c21a96b29e15013416b as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/305ccb6b5e114c21a96b29e15013416b 2024-11-21T03:40:36,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/305ccb6b5e114c21a96b29e15013416b, entries=200, sequenceid=128, filesize=38.6 K 2024-11-21T03:40:36,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/81f6c39659d44b21b852dabd3fe61367 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/81f6c39659d44b21b852dabd3fe61367 2024-11-21T03:40:36,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/81f6c39659d44b21b852dabd3fe61367, entries=150, sequenceid=128, filesize=11.7 K 2024-11-21T03:40:36,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/ee8e3b9f7f35461e9dcd9320c4dce2fe as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/ee8e3b9f7f35461e9dcd9320c4dce2fe 2024-11-21T03:40:36,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/ee8e3b9f7f35461e9dcd9320c4dce2fe, entries=150, sequenceid=128, filesize=11.7 K 2024-11-21T03:40:36,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 9a822f4697b07ce815f004ac9ea14bb2 in 1495ms, sequenceid=128, compaction requested=false 2024-11-21T03:40:36,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:37,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:37,040 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:37,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a822f4697b07ce815f004ac9ea14bb2 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T03:40:37,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c50091405db045b7ad48bc9d51b30a7f_9a822f4697b07ce815f004ac9ea14bb2 is 50, key is test_row_0/A:col10/1732160435518/Put/seqid=0 2024-11-21T03:40:37,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T03:40:37,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741993_1169 (size=14794) 2024-11-21T03:40:37,072 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:37,079 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c50091405db045b7ad48bc9d51b30a7f_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c50091405db045b7ad48bc9d51b30a7f_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:37,080 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/2d47c0b5d4dd43ebaa5c9825740efc95, store: [table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:37,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/2d47c0b5d4dd43ebaa5c9825740efc95 is 175, key is test_row_0/A:col10/1732160435518/Put/seqid=0 2024-11-21T03:40:37,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T03:40:37,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:37,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:37,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:37,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:37,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:37,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:37,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:37,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160497104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:37,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:37,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741994_1170 (size=39749) 2024-11-21T03:40:37,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:37,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160497107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:37,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:37,116 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=143, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/2d47c0b5d4dd43ebaa5c9825740efc95 2024-11-21T03:40:37,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:37,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160497115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:37,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:37,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/b1986deddacd46e7ad80e933ae95fe7b is 50, key is test_row_0/B:col10/1732160435518/Put/seqid=0 2024-11-21T03:40:37,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741995_1171 (size=12151) 2024-11-21T03:40:37,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/b1986deddacd46e7ad80e933ae95fe7b 2024-11-21T03:40:37,180 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/02c709253f014dc1b8acdc9aebf3ec2f is 50, key is test_row_0/C:col10/1732160435518/Put/seqid=0 2024-11-21T03:40:37,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741996_1172 (size=12151) 2024-11-21T03:40:37,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=143 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/02c709253f014dc1b8acdc9aebf3ec2f 2024-11-21T03:40:37,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/2d47c0b5d4dd43ebaa5c9825740efc95 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/2d47c0b5d4dd43ebaa5c9825740efc95 2024-11-21T03:40:37,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/2d47c0b5d4dd43ebaa5c9825740efc95, entries=200, sequenceid=143, filesize=38.8 K 2024-11-21T03:40:37,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/b1986deddacd46e7ad80e933ae95fe7b as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/b1986deddacd46e7ad80e933ae95fe7b 2024-11-21T03:40:37,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/b1986deddacd46e7ad80e933ae95fe7b, entries=150, sequenceid=143, filesize=11.9 K 2024-11-21T03:40:37,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/02c709253f014dc1b8acdc9aebf3ec2f as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/02c709253f014dc1b8acdc9aebf3ec2f 2024-11-21T03:40:37,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160497217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:37,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:37,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160497217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:37,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:37,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/02c709253f014dc1b8acdc9aebf3ec2f, entries=150, sequenceid=143, filesize=11.9 K 2024-11-21T03:40:37,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 9a822f4697b07ce815f004ac9ea14bb2 in 188ms, sequenceid=143, compaction requested=true 2024-11-21T03:40:37,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:37,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a822f4697b07ce815f004ac9ea14bb2:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:40:37,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:37,229 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:37,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a822f4697b07ce815f004ac9ea14bb2:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:40:37,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:37,229 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:37,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a822f4697b07ce815f004ac9ea14bb2:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:40:37,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:37,231 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:37,231 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110493 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:37,231 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 9a822f4697b07ce815f004ac9ea14bb2/A is initiating minor compaction (all files) 2024-11-21T03:40:37,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a822f4697b07ce815f004ac9ea14bb2 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T03:40:37,231 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a822f4697b07ce815f004ac9ea14bb2/A in TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:37,231 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/f66f90cdc25c40d4a017e7a1b3178fb5, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/305ccb6b5e114c21a96b29e15013416b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/2d47c0b5d4dd43ebaa5c9825740efc95] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp, totalSize=107.9 K 2024-11-21T03:40:37,231 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:37,231 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. files: [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/f66f90cdc25c40d4a017e7a1b3178fb5, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/305ccb6b5e114c21a96b29e15013416b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/2d47c0b5d4dd43ebaa5c9825740efc95] 2024-11-21T03:40:37,232 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:37,232 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 9a822f4697b07ce815f004ac9ea14bb2/B is initiating minor compaction (all files) 2024-11-21T03:40:37,232 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a822f4697b07ce815f004ac9ea14bb2/B in TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:37,233 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/a0fb333243a24192b010c3ef4843a657, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/81f6c39659d44b21b852dabd3fe61367, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/b1986deddacd46e7ad80e933ae95fe7b] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp, totalSize=35.5 K 2024-11-21T03:40:37,233 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting f66f90cdc25c40d4a017e7a1b3178fb5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732160434755 2024-11-21T03:40:37,234 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting a0fb333243a24192b010c3ef4843a657, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732160434755 2024-11-21T03:40:37,234 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 305ccb6b5e114c21a96b29e15013416b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732160434823 2024-11-21T03:40:37,236 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 81f6c39659d44b21b852dabd3fe61367, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732160434823 2024-11-21T03:40:37,236 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d47c0b5d4dd43ebaa5c9825740efc95, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732160435513 2024-11-21T03:40:37,237 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting b1986deddacd46e7ad80e933ae95fe7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732160435513 2024-11-21T03:40:37,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T03:40:37,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:37,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:37,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:37,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:37,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:37,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ffb3e052f0f942eabaedfa42ad80fd61_9a822f4697b07ce815f004ac9ea14bb2 is 50, key is test_row_0/A:col10/1732160437229/Put/seqid=0 2024-11-21T03:40:37,269 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:37,269 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a822f4697b07ce815f004ac9ea14bb2#B#compaction#148 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:37,270 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/3a3659c1af094445ab396368c39072c3 is 50, key is test_row_0/B:col10/1732160435518/Put/seqid=0 2024-11-21T03:40:37,275 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121b52a915feaad46519decaef972dfb3e0_9a822f4697b07ce815f004ac9ea14bb2 store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:37,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:37,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160497273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:37,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:37,278 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121b52a915feaad46519decaef972dfb3e0_9a822f4697b07ce815f004ac9ea14bb2, store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:37,278 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b52a915feaad46519decaef972dfb3e0_9a822f4697b07ce815f004ac9ea14bb2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:37,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741997_1173 (size=14794) 2024-11-21T03:40:37,293 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:37,298 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ffb3e052f0f942eabaedfa42ad80fd61_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ffb3e052f0f942eabaedfa42ad80fd61_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:37,300 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/5c48457436a94fad9fa45ba16e68ff4d, store: [table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:37,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/5c48457436a94fad9fa45ba16e68ff4d is 175, key is test_row_0/A:col10/1732160437229/Put/seqid=0 2024-11-21T03:40:37,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741998_1174 (size=12493) 2024-11-21T03:40:37,328 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/3a3659c1af094445ab396368c39072c3 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/3a3659c1af094445ab396368c39072c3 2024-11-21T03:40:37,336 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a822f4697b07ce815f004ac9ea14bb2/B of 9a822f4697b07ce815f004ac9ea14bb2 into 3a3659c1af094445ab396368c39072c3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:37,336 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:37,336 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., storeName=9a822f4697b07ce815f004ac9ea14bb2/B, priority=13, startTime=1732160437229; duration=0sec 2024-11-21T03:40:37,336 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:37,336 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a822f4697b07ce815f004ac9ea14bb2:B 2024-11-21T03:40:37,337 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:37,339 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:37,339 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 9a822f4697b07ce815f004ac9ea14bb2/C is initiating minor compaction (all files) 2024-11-21T03:40:37,339 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a822f4697b07ce815f004ac9ea14bb2/C in TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:37,339 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/f7b533b213884b1f84e3c794495d162a, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/ee8e3b9f7f35461e9dcd9320c4dce2fe, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/02c709253f014dc1b8acdc9aebf3ec2f] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp, totalSize=35.5 K 2024-11-21T03:40:37,340 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting f7b533b213884b1f84e3c794495d162a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=101, earliestPutTs=1732160434755 2024-11-21T03:40:37,340 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting ee8e3b9f7f35461e9dcd9320c4dce2fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1732160434823 2024-11-21T03:40:37,341 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 02c709253f014dc1b8acdc9aebf3ec2f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732160435513 2024-11-21T03:40:37,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742000_1176 (size=39749) 2024-11-21T03:40:37,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073741999_1175 (size=4469) 2024-11-21T03:40:37,359 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a822f4697b07ce815f004ac9ea14bb2#A#compaction#149 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:37,360 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/d176cba569bd4f71b19c71ce95c8d589 is 175, key is test_row_0/A:col10/1732160435518/Put/seqid=0 2024-11-21T03:40:37,373 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a822f4697b07ce815f004ac9ea14bb2#C#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:37,373 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/f40e21eb0340418eb12c1c2fd7e20ae3 is 50, key is test_row_0/C:col10/1732160435518/Put/seqid=0 2024-11-21T03:40:37,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:37,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160497387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:37,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:37,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T03:40:37,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:37,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:37,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:37,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:37,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:37,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:37,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742001_1177 (size=31447) 2024-11-21T03:40:37,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742002_1178 (size=12493) 2024-11-21T03:40:37,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:37,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160497427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:37,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:37,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:37,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160497427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:37,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:37,432 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/f40e21eb0340418eb12c1c2fd7e20ae3 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/f40e21eb0340418eb12c1c2fd7e20ae3 2024-11-21T03:40:37,441 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a822f4697b07ce815f004ac9ea14bb2/C of 9a822f4697b07ce815f004ac9ea14bb2 into f40e21eb0340418eb12c1c2fd7e20ae3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:37,441 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:37,441 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., storeName=9a822f4697b07ce815f004ac9ea14bb2/C, priority=13, startTime=1732160437229; duration=0sec 2024-11-21T03:40:37,442 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:37,442 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a822f4697b07ce815f004ac9ea14bb2:C 2024-11-21T03:40:37,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T03:40:37,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:37,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:37,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:37,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:37,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:37,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:37,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:37,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160497597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:37,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:37,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:37,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160497687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:37,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:37,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160497688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:37,692 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_1' in 9a822f4697b07ce815f004ac9ea14bb2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:40:37,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:37,693 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_2' in 9a822f4697b07ce815f004ac9ea14bb2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:40:37,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:37,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T03:40:37,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:37,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:37,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:37,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:37,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:37,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:37,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:37,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160497741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:37,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:37,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160497741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:37,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:37,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:37,748 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/5c48457436a94fad9fa45ba16e68ff4d 2024-11-21T03:40:37,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/97740e7a533d48d48f9545510bbd40fc is 50, key is test_row_0/B:col10/1732160437229/Put/seqid=0 2024-11-21T03:40:37,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742003_1179 (size=12151) 2024-11-21T03:40:37,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/97740e7a533d48d48f9545510bbd40fc 2024-11-21T03:40:37,817 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/d176cba569bd4f71b19c71ce95c8d589 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/d176cba569bd4f71b19c71ce95c8d589 2024-11-21T03:40:37,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/282b29e526904da4b6e70134a2f5f2e0 is 50, key is test_row_0/C:col10/1732160437229/Put/seqid=0 2024-11-21T03:40:37,825 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a822f4697b07ce815f004ac9ea14bb2/A of 9a822f4697b07ce815f004ac9ea14bb2 into d176cba569bd4f71b19c71ce95c8d589(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:37,826 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:37,826 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., storeName=9a822f4697b07ce815f004ac9ea14bb2/A, priority=13, startTime=1732160437229; duration=0sec 2024-11-21T03:40:37,826 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:37,826 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a822f4697b07ce815f004ac9ea14bb2:A 2024-11-21T03:40:37,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T03:40:37,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:37,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:37,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:37,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:37,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:37,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:37,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742004_1180 (size=12151) 2024-11-21T03:40:37,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/282b29e526904da4b6e70134a2f5f2e0 2024-11-21T03:40:37,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/5c48457436a94fad9fa45ba16e68ff4d as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/5c48457436a94fad9fa45ba16e68ff4d 2024-11-21T03:40:37,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/5c48457436a94fad9fa45ba16e68ff4d, entries=200, sequenceid=169, filesize=38.8 K 2024-11-21T03:40:37,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/97740e7a533d48d48f9545510bbd40fc as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/97740e7a533d48d48f9545510bbd40fc 2024-11-21T03:40:37,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/97740e7a533d48d48f9545510bbd40fc, entries=150, sequenceid=169, filesize=11.9 K 2024-11-21T03:40:37,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/282b29e526904da4b6e70134a2f5f2e0 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/282b29e526904da4b6e70134a2f5f2e0 2024-11-21T03:40:37,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160497907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:37,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:37,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:37,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/282b29e526904da4b6e70134a2f5f2e0, entries=150, sequenceid=169, filesize=11.9 K 2024-11-21T03:40:37,917 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 9a822f4697b07ce815f004ac9ea14bb2 in 686ms, sequenceid=169, compaction requested=false 2024-11-21T03:40:37,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:38,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T03:40:38,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:38,015 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 9a822f4697b07ce815f004ac9ea14bb2 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-21T03:40:38,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216dd2c2faad3a476fb119a408d4b159a9_9a822f4697b07ce815f004ac9ea14bb2 is 50, key is test_row_0/A:col10/1732160437269/Put/seqid=0 2024-11-21T03:40:38,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742005_1181 (size=12304) 2024-11-21T03:40:38,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:38,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:38,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:38,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160498398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:38,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:38,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:38,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:38,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160498403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:38,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:38,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:38,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:38,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:38,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160498416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:38,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:38,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:38,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:38,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:38,447 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216dd2c2faad3a476fb119a408d4b159a9_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216dd2c2faad3a476fb119a408d4b159a9_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:38,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/477547b4a3064de5886bbc63ef39691c, store: [table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:38,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/477547b4a3064de5886bbc63ef39691c is 175, key is test_row_0/A:col10/1732160437269/Put/seqid=0 2024-11-21T03:40:38,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742006_1182 (size=31105) 2024-11-21T03:40:38,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160498507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:38,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:38,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160498517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:38,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:38,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:38,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:38,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160498717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:38,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160498727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:38,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:38,898 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/477547b4a3064de5886bbc63ef39691c 2024-11-21T03:40:38,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/735ec1b9a2434ebe99a97dbd8d1bf4c9 is 50, key is test_row_0/B:col10/1732160437269/Put/seqid=0 2024-11-21T03:40:38,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742007_1183 (size=12151) 2024-11-21T03:40:39,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:39,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160499027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:39,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:39,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:39,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:39,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:39,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160499037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:39,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:39,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:39,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T03:40:39,334 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/735ec1b9a2434ebe99a97dbd8d1bf4c9 2024-11-21T03:40:39,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/d229c97300cc4e6bb56b1c24fed9dd1b is 50, key is test_row_0/C:col10/1732160437269/Put/seqid=0 2024-11-21T03:40:39,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742008_1184 (size=12151) 2024-11-21T03:40:39,353 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/d229c97300cc4e6bb56b1c24fed9dd1b 2024-11-21T03:40:39,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/477547b4a3064de5886bbc63ef39691c as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/477547b4a3064de5886bbc63ef39691c 2024-11-21T03:40:39,366 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/477547b4a3064de5886bbc63ef39691c, entries=150, sequenceid=182, filesize=30.4 K 2024-11-21T03:40:39,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/735ec1b9a2434ebe99a97dbd8d1bf4c9 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/735ec1b9a2434ebe99a97dbd8d1bf4c9 2024-11-21T03:40:39,375 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/735ec1b9a2434ebe99a97dbd8d1bf4c9, entries=150, sequenceid=182, filesize=11.9 K 2024-11-21T03:40:39,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/d229c97300cc4e6bb56b1c24fed9dd1b as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/d229c97300cc4e6bb56b1c24fed9dd1b 2024-11-21T03:40:39,382 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/d229c97300cc4e6bb56b1c24fed9dd1b, entries=150, sequenceid=182, filesize=11.9 K 2024-11-21T03:40:39,383 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 9a822f4697b07ce815f004ac9ea14bb2 in 1368ms, sequenceid=182, compaction requested=true 2024-11-21T03:40:39,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:39,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:39,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-21T03:40:39,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-21T03:40:39,388 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-21T03:40:39,388 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.4730 sec 2024-11-21T03:40:39,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 4.4800 sec 2024-11-21T03:40:39,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:39,439 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:39,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a822f4697b07ce815f004ac9ea14bb2 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-21T03:40:39,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a79b0620174b44399359ffd2917d0bb0_9a822f4697b07ce815f004ac9ea14bb2 is 50, key is test_row_0/A:col10/1732160438401/Put/seqid=0 2024-11-21T03:40:39,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160499478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:39,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:39,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:39,482 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:39,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742009_1185 (size=12304) 2024-11-21T03:40:39,503 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:39,509 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a79b0620174b44399359ffd2917d0bb0_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a79b0620174b44399359ffd2917d0bb0_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:39,512 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/7b48c86e313c44c99a7856986576695e, store: [table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:39,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/7b48c86e313c44c99a7856986576695e is 175, key is test_row_0/A:col10/1732160438401/Put/seqid=0 2024-11-21T03:40:39,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742010_1186 (size=31105) 2024-11-21T03:40:39,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:39,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160499537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:39,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:39,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:39,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:39,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:39,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160499547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:39,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:39,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:39,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:39,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:39,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160499587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:39,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:39,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160499707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:39,711 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_1' in 9a822f4697b07ce815f004ac9ea14bb2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:40:39,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:39,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:39,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:39,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:39,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160499717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:39,718 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_2' in 9a822f4697b07ce815f004ac9ea14bb2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:40:39,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:39,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:39,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:39,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:39,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160499797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:39,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:39,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:39,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:39,927 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/7b48c86e313c44c99a7856986576695e 2024-11-21T03:40:39,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/1327fe631bc34b4ca1d701b24681e54c is 50, key is test_row_0/B:col10/1732160438401/Put/seqid=0 2024-11-21T03:40:39,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742011_1187 (size=12151) 2024-11-21T03:40:40,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160500107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:40,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:40,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:40,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:40,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/1327fe631bc34b4ca1d701b24681e54c 2024-11-21T03:40:40,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/f56c824f7e7c45348858cc22ac4bb769 is 50, key is test_row_0/C:col10/1732160438401/Put/seqid=0 2024-11-21T03:40:40,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742012_1188 (size=12151) 2024-11-21T03:40:40,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:40,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160500547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:40,548 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_2' in 9a822f4697b07ce815f004ac9ea14bb2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:40:40,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:40,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:40,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:40,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:40,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160500557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:40,558 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_0' in 9a822f4697b07ce815f004ac9ea14bb2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:40:40,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:40,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:40,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:40,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:40,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160500617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:40,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:40,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:40,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:40,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/f56c824f7e7c45348858cc22ac4bb769 2024-11-21T03:40:40,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/7b48c86e313c44c99a7856986576695e as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/7b48c86e313c44c99a7856986576695e 2024-11-21T03:40:40,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/7b48c86e313c44c99a7856986576695e, entries=150, sequenceid=209, filesize=30.4 K 2024-11-21T03:40:40,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/1327fe631bc34b4ca1d701b24681e54c as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/1327fe631bc34b4ca1d701b24681e54c 2024-11-21T03:40:40,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/1327fe631bc34b4ca1d701b24681e54c, entries=150, sequenceid=209, filesize=11.9 K 2024-11-21T03:40:40,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/f56c824f7e7c45348858cc22ac4bb769 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/f56c824f7e7c45348858cc22ac4bb769 2024-11-21T03:40:40,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/f56c824f7e7c45348858cc22ac4bb769, entries=150, sequenceid=209, filesize=11.9 K 2024-11-21T03:40:40,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 9a822f4697b07ce815f004ac9ea14bb2 in 1391ms, sequenceid=209, compaction requested=true 2024-11-21T03:40:40,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:40,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a822f4697b07ce815f004ac9ea14bb2:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:40:40,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:40,830 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:40:40,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a822f4697b07ce815f004ac9ea14bb2:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:40:40,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:40,830 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:40:40,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a822f4697b07ce815f004ac9ea14bb2:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:40:40,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:40,832 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:40:40,832 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 9a822f4697b07ce815f004ac9ea14bb2/B is initiating minor compaction (all files) 2024-11-21T03:40:40,832 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a822f4697b07ce815f004ac9ea14bb2/B in TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:40,832 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/3a3659c1af094445ab396368c39072c3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/97740e7a533d48d48f9545510bbd40fc, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/735ec1b9a2434ebe99a97dbd8d1bf4c9, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/1327fe631bc34b4ca1d701b24681e54c] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp, totalSize=47.8 K 2024-11-21T03:40:40,832 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133406 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:40:40,832 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 9a822f4697b07ce815f004ac9ea14bb2/A is initiating minor compaction (all files) 2024-11-21T03:40:40,833 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a822f4697b07ce815f004ac9ea14bb2/A in TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:40,833 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/d176cba569bd4f71b19c71ce95c8d589, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/5c48457436a94fad9fa45ba16e68ff4d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/477547b4a3064de5886bbc63ef39691c, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/7b48c86e313c44c99a7856986576695e] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp, totalSize=130.3 K 2024-11-21T03:40:40,833 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a3659c1af094445ab396368c39072c3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732160435513 2024-11-21T03:40:40,833 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:40,833 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. files: [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/d176cba569bd4f71b19c71ce95c8d589, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/5c48457436a94fad9fa45ba16e68ff4d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/477547b4a3064de5886bbc63ef39691c, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/7b48c86e313c44c99a7856986576695e] 2024-11-21T03:40:40,834 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 97740e7a533d48d48f9545510bbd40fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732160437103 2024-11-21T03:40:40,834 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d176cba569bd4f71b19c71ce95c8d589, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732160435513 2024-11-21T03:40:40,834 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c48457436a94fad9fa45ba16e68ff4d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732160437095 2024-11-21T03:40:40,834 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 735ec1b9a2434ebe99a97dbd8d1bf4c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732160437262 2024-11-21T03:40:40,835 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 477547b4a3064de5886bbc63ef39691c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732160437262 2024-11-21T03:40:40,835 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 1327fe631bc34b4ca1d701b24681e54c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732160438357 2024-11-21T03:40:40,836 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b48c86e313c44c99a7856986576695e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732160438357 2024-11-21T03:40:40,846 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:40,846 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a822f4697b07ce815f004ac9ea14bb2#B#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:40,846 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/22b32f1fd2c1443d89bc83420fb0e6c0 is 50, key is test_row_0/B:col10/1732160438401/Put/seqid=0 2024-11-21T03:40:40,848 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121997d83ba032746f0ad02509ecf68f562_9a822f4697b07ce815f004ac9ea14bb2 store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:40,851 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121997d83ba032746f0ad02509ecf68f562_9a822f4697b07ce815f004ac9ea14bb2, store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:40,851 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121997d83ba032746f0ad02509ecf68f562_9a822f4697b07ce815f004ac9ea14bb2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:40,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742013_1189 (size=12629) 2024-11-21T03:40:40,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742014_1190 (size=4469) 2024-11-21T03:40:41,109 INFO [master/fdc0a393186c:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-21T03:40:41,109 INFO [master/fdc0a393186c:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-21T03:40:41,271 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/22b32f1fd2c1443d89bc83420fb0e6c0 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/22b32f1fd2c1443d89bc83420fb0e6c0 2024-11-21T03:40:41,273 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a822f4697b07ce815f004ac9ea14bb2#A#compaction#160 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:41,273 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/c122007c522e49f18f594c218b7c8601 is 175, key is test_row_0/A:col10/1732160438401/Put/seqid=0 2024-11-21T03:40:41,279 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9a822f4697b07ce815f004ac9ea14bb2/B of 9a822f4697b07ce815f004ac9ea14bb2 into 22b32f1fd2c1443d89bc83420fb0e6c0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:41,279 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:41,279 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., storeName=9a822f4697b07ce815f004ac9ea14bb2/B, priority=12, startTime=1732160440830; duration=0sec 2024-11-21T03:40:41,280 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:41,280 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a822f4697b07ce815f004ac9ea14bb2:B 2024-11-21T03:40:41,280 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:40:41,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742015_1191 (size=31583) 2024-11-21T03:40:41,288 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:40:41,288 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 9a822f4697b07ce815f004ac9ea14bb2/C is initiating minor compaction (all files) 2024-11-21T03:40:41,288 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a822f4697b07ce815f004ac9ea14bb2/C in TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:41,288 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/f40e21eb0340418eb12c1c2fd7e20ae3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/282b29e526904da4b6e70134a2f5f2e0, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/d229c97300cc4e6bb56b1c24fed9dd1b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/f56c824f7e7c45348858cc22ac4bb769] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp, totalSize=47.8 K 2024-11-21T03:40:41,289 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting f40e21eb0340418eb12c1c2fd7e20ae3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=143, earliestPutTs=1732160435513 2024-11-21T03:40:41,289 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 282b29e526904da4b6e70134a2f5f2e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732160437103 2024-11-21T03:40:41,290 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting d229c97300cc4e6bb56b1c24fed9dd1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1732160437262 2024-11-21T03:40:41,291 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting f56c824f7e7c45348858cc22ac4bb769, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732160438357 2024-11-21T03:40:41,294 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/c122007c522e49f18f594c218b7c8601 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/c122007c522e49f18f594c218b7c8601 2024-11-21T03:40:41,302 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9a822f4697b07ce815f004ac9ea14bb2/A of 9a822f4697b07ce815f004ac9ea14bb2 into c122007c522e49f18f594c218b7c8601(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:41,302 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:41,302 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., storeName=9a822f4697b07ce815f004ac9ea14bb2/A, priority=12, startTime=1732160440830; duration=0sec 2024-11-21T03:40:41,302 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:41,302 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a822f4697b07ce815f004ac9ea14bb2:A 2024-11-21T03:40:41,307 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a822f4697b07ce815f004ac9ea14bb2#C#compaction#161 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:41,307 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/524e4741d93c4b45a56c38669cc2a7d6 is 50, key is test_row_0/C:col10/1732160438401/Put/seqid=0 2024-11-21T03:40:41,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742016_1192 (size=12629) 2024-11-21T03:40:41,328 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/524e4741d93c4b45a56c38669cc2a7d6 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/524e4741d93c4b45a56c38669cc2a7d6 2024-11-21T03:40:41,337 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9a822f4697b07ce815f004ac9ea14bb2/C of 9a822f4697b07ce815f004ac9ea14bb2 into 524e4741d93c4b45a56c38669cc2a7d6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:41,337 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:41,337 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., storeName=9a822f4697b07ce815f004ac9ea14bb2/C, priority=12, startTime=1732160440830; duration=0sec 2024-11-21T03:40:41,337 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:41,337 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a822f4697b07ce815f004ac9ea14bb2:C 2024-11-21T03:40:41,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:41,631 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:41,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a822f4697b07ce815f004ac9ea14bb2 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:40:41,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bf21173b9f304886bb372b24de0c4b61_9a822f4697b07ce815f004ac9ea14bb2 is 50, key is test_row_0/A:col10/1732160441628/Put/seqid=0 2024-11-21T03:40:41,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742017_1193 (size=14794) 2024-11-21T03:40:41,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160501733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:41,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:41,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:41,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:41,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:41,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160501846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:41,847 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:42,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:42,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160502057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:42,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:42,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:42,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:42,093 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:42,097 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bf21173b9f304886bb372b24de0c4b61_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bf21173b9f304886bb372b24de0c4b61_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:42,098 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/86ff4be961dd420e9f36fb99a492460e, store: [table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:42,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/86ff4be961dd420e9f36fb99a492460e is 175, key is test_row_0/A:col10/1732160441628/Put/seqid=0 2024-11-21T03:40:42,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742018_1194 (size=39749) 2024-11-21T03:40:42,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:42,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160502366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:42,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:42,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:42,367 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:42,506 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=223, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/86ff4be961dd420e9f36fb99a492460e 2024-11-21T03:40:42,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/c75eb17c981c4c6ba90457539f1efda5 is 50, key is test_row_0/B:col10/1732160441628/Put/seqid=0 2024-11-21T03:40:42,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742019_1195 (size=12151) 2024-11-21T03:40:42,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:42,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160502557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:42,558 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_2' in 9a822f4697b07ce815f004ac9ea14bb2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:40:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:42,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:42,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160502567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:42,569 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_0' in 9a822f4697b07ce815f004ac9ea14bb2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:40:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:42,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160502877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:42,879 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:42,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/c75eb17c981c4c6ba90457539f1efda5 2024-11-21T03:40:42,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/c1ecb486c6904b639f6cedc510849ab5 is 50, key is test_row_0/C:col10/1732160441628/Put/seqid=0 2024-11-21T03:40:42,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742020_1196 (size=12151) 2024-11-21T03:40:43,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T03:40:43,097 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:43,098 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:40:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-21T03:40:43,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T03:40:43,100 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:40:43,101 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:40:43,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:40:43,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T03:40:43,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T03:40:43,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:43,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:43,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:43,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:43,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:43,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:43,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/c1ecb486c6904b639f6cedc510849ab5 2024-11-21T03:40:43,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/86ff4be961dd420e9f36fb99a492460e as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/86ff4be961dd420e9f36fb99a492460e 2024-11-21T03:40:43,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/86ff4be961dd420e9f36fb99a492460e, entries=200, sequenceid=223, filesize=38.8 K 2024-11-21T03:40:43,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/c75eb17c981c4c6ba90457539f1efda5 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/c75eb17c981c4c6ba90457539f1efda5 2024-11-21T03:40:43,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/c75eb17c981c4c6ba90457539f1efda5, entries=150, sequenceid=223, filesize=11.9 K 2024-11-21T03:40:43,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/c1ecb486c6904b639f6cedc510849ab5 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/c1ecb486c6904b639f6cedc510849ab5 2024-11-21T03:40:43,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/c1ecb486c6904b639f6cedc510849ab5, entries=150, sequenceid=223, filesize=11.9 K 2024-11-21T03:40:43,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 9a822f4697b07ce815f004ac9ea14bb2 in 1721ms, sequenceid=223, compaction requested=false 2024-11-21T03:40:43,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:43,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T03:40:43,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:43,409 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 9a822f4697b07ce815f004ac9ea14bb2 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T03:40:43,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T03:40:43,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a99252b3bab74c1cb9fa8666bb0467aa_9a822f4697b07ce815f004ac9ea14bb2 is 50, key is test_row_0/A:col10/1732160441731/Put/seqid=0 2024-11-21T03:40:43,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742021_1197 (size=12304) 2024-11-21T03:40:43,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:43,431 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a99252b3bab74c1cb9fa8666bb0467aa_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a99252b3bab74c1cb9fa8666bb0467aa_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:43,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/f6ec49fc7a104ff89db5b6ccee8b7f35, store: [table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:43,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/f6ec49fc7a104ff89db5b6ccee8b7f35 is 175, key is test_row_0/A:col10/1732160441731/Put/seqid=0 2024-11-21T03:40:43,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742022_1198 (size=31105) 2024-11-21T03:40:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T03:40:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:43,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:43,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:43,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160503743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:43,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:43,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160503744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:43,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:43,839 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/f6ec49fc7a104ff89db5b6ccee8b7f35 2024-11-21T03:40:43,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/dccbfd9f98154d7a8d085fdf03a6394b is 50, key is test_row_0/B:col10/1732160441731/Put/seqid=0 2024-11-21T03:40:43,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742023_1199 (size=12151) 2024-11-21T03:40:43,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:43,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160503857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:43,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:43,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:43,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:43,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160503857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:43,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:43,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:43,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:43,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:43,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:43,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160503886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:43,889 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_0' in 9a822f4697b07ce815f004ac9ea14bb2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:40:43,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:43,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:43,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:44,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:44,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160504067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:44,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:44,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160504067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:44,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:44,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T03:40:44,251 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/dccbfd9f98154d7a8d085fdf03a6394b 2024-11-21T03:40:44,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/a75728a5b94444129201d3fd95d21bda is 50, key is test_row_0/C:col10/1732160441731/Put/seqid=0 2024-11-21T03:40:44,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742024_1200 (size=12151) 2024-11-21T03:40:44,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160504377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:44,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160504377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:44,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:44,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:44,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:44,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:44,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:44,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:44,701 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/a75728a5b94444129201d3fd95d21bda 2024-11-21T03:40:44,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/f6ec49fc7a104ff89db5b6ccee8b7f35 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/f6ec49fc7a104ff89db5b6ccee8b7f35 2024-11-21T03:40:44,714 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/f6ec49fc7a104ff89db5b6ccee8b7f35, entries=150, sequenceid=248, filesize=30.4 K 2024-11-21T03:40:44,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/dccbfd9f98154d7a8d085fdf03a6394b as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/dccbfd9f98154d7a8d085fdf03a6394b 2024-11-21T03:40:44,722 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/dccbfd9f98154d7a8d085fdf03a6394b, entries=150, sequenceid=248, filesize=11.9 K 2024-11-21T03:40:44,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/a75728a5b94444129201d3fd95d21bda as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/a75728a5b94444129201d3fd95d21bda 2024-11-21T03:40:44,731 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/a75728a5b94444129201d3fd95d21bda, entries=150, sequenceid=248, filesize=11.9 K 2024-11-21T03:40:44,732 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 9a822f4697b07ce815f004ac9ea14bb2 in 1323ms, sequenceid=248, compaction requested=true 2024-11-21T03:40:44,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:44,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:44,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-21T03:40:44,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-21T03:40:44,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-21T03:40:44,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6330 sec 2024-11-21T03:40:44,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.6390 sec 2024-11-21T03:40:44,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:44,904 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:44,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a822f4697b07ce815f004ac9ea14bb2 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T03:40:44,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212a84204128654629802c988773ee4726_9a822f4697b07ce815f004ac9ea14bb2 is 50, key is test_row_0/A:col10/1732160443742/Put/seqid=0 2024-11-21T03:40:44,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742025_1201 (size=17484) 2024-11-21T03:40:44,931 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:44,942 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212a84204128654629802c988773ee4726_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212a84204128654629802c988773ee4726_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:44,943 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/5d87d39a7b1c46029ea92ed14ebd16b3, store: [table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:44,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/5d87d39a7b1c46029ea92ed14ebd16b3 is 175, key is test_row_0/A:col10/1732160443742/Put/seqid=0 2024-11-21T03:40:44,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742026_1202 (size=48589) 2024-11-21T03:40:44,956 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=261, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/5d87d39a7b1c46029ea92ed14ebd16b3 2024-11-21T03:40:44,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/91a6cf96ae604429b467b1a7671095e1 is 50, key is test_row_0/B:col10/1732160443742/Put/seqid=0 2024-11-21T03:40:44,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742027_1203 (size=12301) 2024-11-21T03:40:45,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:45,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160504998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:45,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:45,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160505000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:45,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:45,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:45,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160505107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:45,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160505108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:45,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:45,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:45,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:45,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:45,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:45,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:45,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T03:40:45,247 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:45,247 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:40:45,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-21T03:40:45,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T03:40:45,250 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:40:45,252 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:40:45,252 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:40:45,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:45,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160505316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:45,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:45,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:45,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:45,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:45,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160505317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:45,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:45,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:45,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:45,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T03:40:45,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/91a6cf96ae604429b467b1a7671095e1 2024-11-21T03:40:45,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/8150c3401fe643fb9232345bad0025c6 is 50, key is test_row_0/C:col10/1732160443742/Put/seqid=0 2024-11-21T03:40:45,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742028_1204 (size=12301) 2024-11-21T03:40:45,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-21T03:40:45,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:45,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:45,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:45,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:45,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:45,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:45,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-21T03:40:45,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:45,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:45,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:45,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:45,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:45,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T03:40:45,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:45,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160505636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:45,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:45,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160505636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:45,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:45,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:45,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:45,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:45,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-21T03:40:45,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:45,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:45,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:45,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:45,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:45,801 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/8150c3401fe643fb9232345bad0025c6 2024-11-21T03:40:45,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/5d87d39a7b1c46029ea92ed14ebd16b3 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/5d87d39a7b1c46029ea92ed14ebd16b3 2024-11-21T03:40:45,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/5d87d39a7b1c46029ea92ed14ebd16b3, entries=250, sequenceid=261, filesize=47.5 K 2024-11-21T03:40:45,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/91a6cf96ae604429b467b1a7671095e1 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/91a6cf96ae604429b467b1a7671095e1 2024-11-21T03:40:45,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/91a6cf96ae604429b467b1a7671095e1, entries=150, sequenceid=261, filesize=12.0 K 2024-11-21T03:40:45,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/8150c3401fe643fb9232345bad0025c6 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/8150c3401fe643fb9232345bad0025c6 2024-11-21T03:40:45,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/8150c3401fe643fb9232345bad0025c6, entries=150, sequenceid=261, filesize=12.0 K 2024-11-21T03:40:45,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 9a822f4697b07ce815f004ac9ea14bb2 in 932ms, sequenceid=261, compaction requested=true 2024-11-21T03:40:45,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:45,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a822f4697b07ce815f004ac9ea14bb2:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:40:45,836 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:40:45,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:45,838 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:40:45,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a822f4697b07ce815f004ac9ea14bb2:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:40:45,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:45,838 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151026 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:40:45,839 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 9a822f4697b07ce815f004ac9ea14bb2/A is initiating minor compaction (all files) 2024-11-21T03:40:45,839 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a822f4697b07ce815f004ac9ea14bb2/A in TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:45,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a822f4697b07ce815f004ac9ea14bb2:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:40:45,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:45,839 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/c122007c522e49f18f594c218b7c8601, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/86ff4be961dd420e9f36fb99a492460e, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/f6ec49fc7a104ff89db5b6ccee8b7f35, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/5d87d39a7b1c46029ea92ed14ebd16b3] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp, totalSize=147.5 K 2024-11-21T03:40:45,839 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:45,839 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. files: [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/c122007c522e49f18f594c218b7c8601, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/86ff4be961dd420e9f36fb99a492460e, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/f6ec49fc7a104ff89db5b6ccee8b7f35, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/5d87d39a7b1c46029ea92ed14ebd16b3] 2024-11-21T03:40:45,839 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:40:45,839 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting c122007c522e49f18f594c218b7c8601, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732160438357 2024-11-21T03:40:45,839 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 9a822f4697b07ce815f004ac9ea14bb2/B is initiating minor compaction (all files) 2024-11-21T03:40:45,840 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a822f4697b07ce815f004ac9ea14bb2/B in TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:45,840 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/22b32f1fd2c1443d89bc83420fb0e6c0, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/c75eb17c981c4c6ba90457539f1efda5, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/dccbfd9f98154d7a8d085fdf03a6394b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/91a6cf96ae604429b467b1a7671095e1] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp, totalSize=48.1 K 2024-11-21T03:40:45,840 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 22b32f1fd2c1443d89bc83420fb0e6c0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732160438357 2024-11-21T03:40:45,841 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86ff4be961dd420e9f36fb99a492460e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732160439464 2024-11-21T03:40:45,842 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting c75eb17c981c4c6ba90457539f1efda5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732160439464 2024-11-21T03:40:45,844 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6ec49fc7a104ff89db5b6ccee8b7f35, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732160441707 2024-11-21T03:40:45,844 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting dccbfd9f98154d7a8d085fdf03a6394b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732160441707 2024-11-21T03:40:45,845 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 91a6cf96ae604429b467b1a7671095e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732160443742 2024-11-21T03:40:45,845 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d87d39a7b1c46029ea92ed14ebd16b3, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732160443738 2024-11-21T03:40:45,865 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a822f4697b07ce815f004ac9ea14bb2#B#compaction#171 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:45,866 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/ff04b1f700c94b1d890a7218ff432c42 is 50, key is test_row_0/B:col10/1732160443742/Put/seqid=0 2024-11-21T03:40:45,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-21T03:40:45,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:45,867 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 9a822f4697b07ce815f004ac9ea14bb2 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T03:40:45,875 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T03:40:45,891 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121ee8bdb91f2b542cca496f2637ef4553f_9a822f4697b07ce815f004ac9ea14bb2 store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:45,893 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121ee8bdb91f2b542cca496f2637ef4553f_9a822f4697b07ce815f004ac9ea14bb2, store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:45,893 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ee8bdb91f2b542cca496f2637ef4553f_9a822f4697b07ce815f004ac9ea14bb2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:45,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216704ef3f956a4c018005929160401d77_9a822f4697b07ce815f004ac9ea14bb2 is 50, key is test_row_0/A:col10/1732160444998/Put/seqid=0 2024-11-21T03:40:45,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:45,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:45,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742029_1205 (size=12915) 2024-11-21T03:40:45,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742030_1206 (size=4469) 2024-11-21T03:40:45,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742031_1207 (size=12454) 2024-11-21T03:40:45,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:45,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160505953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:45,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:45,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:45,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:46,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:46,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160506056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:46,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:46,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160506147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:46,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:46,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:46,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:46,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160506147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:46,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:46,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:46,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:46,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160506267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:46,317 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/ff04b1f700c94b1d890a7218ff432c42 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/ff04b1f700c94b1d890a7218ff432c42 2024-11-21T03:40:46,322 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a822f4697b07ce815f004ac9ea14bb2#A#compaction#172 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:46,323 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/bdecaedec4ed4590998e18b9522503f1 is 175, key is test_row_0/A:col10/1732160443742/Put/seqid=0 2024-11-21T03:40:46,324 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9a822f4697b07ce815f004ac9ea14bb2/B of 9a822f4697b07ce815f004ac9ea14bb2 into ff04b1f700c94b1d890a7218ff432c42(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:46,324 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:46,324 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., storeName=9a822f4697b07ce815f004ac9ea14bb2/B, priority=12, startTime=1732160445838; duration=0sec 2024-11-21T03:40:46,325 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:46,325 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a822f4697b07ce815f004ac9ea14bb2:B 2024-11-21T03:40:46,325 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:40:46,327 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:40:46,327 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 9a822f4697b07ce815f004ac9ea14bb2/C is initiating minor compaction (all files) 2024-11-21T03:40:46,327 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a822f4697b07ce815f004ac9ea14bb2/C in TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:46,328 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/524e4741d93c4b45a56c38669cc2a7d6, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/c1ecb486c6904b639f6cedc510849ab5, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/a75728a5b94444129201d3fd95d21bda, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/8150c3401fe643fb9232345bad0025c6] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp, totalSize=48.1 K 2024-11-21T03:40:46,328 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 524e4741d93c4b45a56c38669cc2a7d6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732160438357 2024-11-21T03:40:46,328 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting c1ecb486c6904b639f6cedc510849ab5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732160439464 2024-11-21T03:40:46,329 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting a75728a5b94444129201d3fd95d21bda, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732160441707 2024-11-21T03:40:46,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:46,329 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8150c3401fe643fb9232345bad0025c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732160443742 2024-11-21T03:40:46,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742032_1208 (size=31869) 2024-11-21T03:40:46,338 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216704ef3f956a4c018005929160401d77_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216704ef3f956a4c018005929160401d77_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:46,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/1ce59c7341054d338d8a7c01a1abb4fe, store: [table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:46,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/1ce59c7341054d338d8a7c01a1abb4fe is 175, key is test_row_0/A:col10/1732160444998/Put/seqid=0 2024-11-21T03:40:46,349 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/bdecaedec4ed4590998e18b9522503f1 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/bdecaedec4ed4590998e18b9522503f1 2024-11-21T03:40:46,358 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9a822f4697b07ce815f004ac9ea14bb2/A of 9a822f4697b07ce815f004ac9ea14bb2 into bdecaedec4ed4590998e18b9522503f1(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:46,358 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:46,358 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., storeName=9a822f4697b07ce815f004ac9ea14bb2/A, priority=12, startTime=1732160445836; duration=0sec 2024-11-21T03:40:46,358 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:46,358 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a822f4697b07ce815f004ac9ea14bb2:A 2024-11-21T03:40:46,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742033_1209 (size=31255) 2024-11-21T03:40:46,385 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/1ce59c7341054d338d8a7c01a1abb4fe 2024-11-21T03:40:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T03:40:46,400 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a822f4697b07ce815f004ac9ea14bb2#C#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:46,400 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/b59e09dd28604825803e990aed41ad33 is 50, key is test_row_0/C:col10/1732160443742/Put/seqid=0 2024-11-21T03:40:46,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/cf4f1d9268994c798c00f5b0ac111781 is 50, key is test_row_0/B:col10/1732160444998/Put/seqid=0 2024-11-21T03:40:46,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742035_1211 (size=12301) 2024-11-21T03:40:46,444 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/cf4f1d9268994c798c00f5b0ac111781 2024-11-21T03:40:46,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742034_1210 (size=12915) 2024-11-21T03:40:46,467 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/b59e09dd28604825803e990aed41ad33 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/b59e09dd28604825803e990aed41ad33 2024-11-21T03:40:46,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/73e3beed7b904a0bade2fc4e4d69a511 is 50, key is test_row_0/C:col10/1732160444998/Put/seqid=0 2024-11-21T03:40:46,476 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 9a822f4697b07ce815f004ac9ea14bb2/C of 9a822f4697b07ce815f004ac9ea14bb2 into b59e09dd28604825803e990aed41ad33(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:46,477 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:46,477 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., storeName=9a822f4697b07ce815f004ac9ea14bb2/C, priority=12, startTime=1732160445839; duration=0sec 2024-11-21T03:40:46,477 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:46,477 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a822f4697b07ce815f004ac9ea14bb2:C 2024-11-21T03:40:46,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742036_1212 (size=12301) 2024-11-21T03:40:46,520 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/73e3beed7b904a0bade2fc4e4d69a511 2024-11-21T03:40:46,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/1ce59c7341054d338d8a7c01a1abb4fe as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/1ce59c7341054d338d8a7c01a1abb4fe 2024-11-21T03:40:46,532 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/1ce59c7341054d338d8a7c01a1abb4fe, entries=150, sequenceid=284, filesize=30.5 K 2024-11-21T03:40:46,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/cf4f1d9268994c798c00f5b0ac111781 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/cf4f1d9268994c798c00f5b0ac111781 2024-11-21T03:40:46,540 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/cf4f1d9268994c798c00f5b0ac111781, entries=150, sequenceid=284, filesize=12.0 K 2024-11-21T03:40:46,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/73e3beed7b904a0bade2fc4e4d69a511 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/73e3beed7b904a0bade2fc4e4d69a511 2024-11-21T03:40:46,548 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/73e3beed7b904a0bade2fc4e4d69a511, entries=150, sequenceid=284, filesize=12.0 K 2024-11-21T03:40:46,549 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 9a822f4697b07ce815f004ac9ea14bb2 in 682ms, sequenceid=284, compaction requested=false 2024-11-21T03:40:46,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:46,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:46,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-21T03:40:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-21T03:40:46,554 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-21T03:40:46,554 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2990 sec 2024-11-21T03:40:46,557 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.3080 sec 2024-11-21T03:40:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:46,570 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:46,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a822f4697b07ce815f004ac9ea14bb2 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T03:40:46,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216a2f3856e4ce458bb113f067854fc218_9a822f4697b07ce815f004ac9ea14bb2 is 50, key is test_row_0/A:col10/1732160445952/Put/seqid=0 2024-11-21T03:40:46,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742037_1213 (size=14994) 2024-11-21T03:40:46,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160506667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:46,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160506667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:46,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:46,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160506667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:46,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:46,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:46,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:46,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:46,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160506787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:46,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:46,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:46,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160506787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:46,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:46,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:46,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:46,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:46,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160506787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:46,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:46,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:46,796 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:46,796 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:46,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:46,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160506997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:46,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:46,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:46,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:47,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:47,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160506997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:47,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:47,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:47,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:47,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:47,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160507007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:47,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:47,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:47,012 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:47,017 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:47,027 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216a2f3856e4ce458bb113f067854fc218_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216a2f3856e4ce458bb113f067854fc218_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:47,030 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/c28b2b01720e4e98bbd8f462f0fe0871, store: [table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:47,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/c28b2b01720e4e98bbd8f462f0fe0871 is 175, key is test_row_0/A:col10/1732160445952/Put/seqid=0 2024-11-21T03:40:47,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742038_1214 (size=39949) 2024-11-21T03:40:47,058 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=301, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/c28b2b01720e4e98bbd8f462f0fe0871 2024-11-21T03:40:47,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/64dc95e6cc9f460583ecda9b591ee16b is 50, key is test_row_0/B:col10/1732160445952/Put/seqid=0 2024-11-21T03:40:47,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742039_1215 (size=12301) 2024-11-21T03:40:47,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/64dc95e6cc9f460583ecda9b591ee16b 2024-11-21T03:40:47,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/a0b7015297df4f678197324511758a25 is 50, key is test_row_0/C:col10/1732160445952/Put/seqid=0 2024-11-21T03:40:47,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742040_1216 (size=12301) 2024-11-21T03:40:47,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:47,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160507167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:47,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:47,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160507167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:47,170 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_2' in 9a822f4697b07ce815f004ac9ea14bb2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:40:47,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:47,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:47,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:47,170 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_2' in 9a822f4697b07ce815f004ac9ea14bb2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:40:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:47,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:47,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160507307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:47,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160507307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:47,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:47,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:47,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:47,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:47,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:47,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:47,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160507317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:47,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:47,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T03:40:47,398 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:47,400 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:40:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-21T03:40:47,404 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:40:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T03:40:47,405 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:40:47,405 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:40:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T03:40:47,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-21T03:40:47,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:47,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:47,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:47,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:47,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:47,566 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/a0b7015297df4f678197324511758a25 2024-11-21T03:40:47,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/c28b2b01720e4e98bbd8f462f0fe0871 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/c28b2b01720e4e98bbd8f462f0fe0871 2024-11-21T03:40:47,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/c28b2b01720e4e98bbd8f462f0fe0871, entries=200, sequenceid=301, filesize=39.0 K 2024-11-21T03:40:47,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/64dc95e6cc9f460583ecda9b591ee16b as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/64dc95e6cc9f460583ecda9b591ee16b 2024-11-21T03:40:47,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/64dc95e6cc9f460583ecda9b591ee16b, entries=150, sequenceid=301, filesize=12.0 K 2024-11-21T03:40:47,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/a0b7015297df4f678197324511758a25 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/a0b7015297df4f678197324511758a25 2024-11-21T03:40:47,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/a0b7015297df4f678197324511758a25, entries=150, sequenceid=301, filesize=12.0 K 2024-11-21T03:40:47,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=140.89 KB/144270 for 9a822f4697b07ce815f004ac9ea14bb2 in 1023ms, sequenceid=301, compaction requested=true 2024-11-21T03:40:47,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:47,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a822f4697b07ce815f004ac9ea14bb2:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:40:47,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:47,593 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:47,593 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:47,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a822f4697b07ce815f004ac9ea14bb2:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:40:47,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:47,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a822f4697b07ce815f004ac9ea14bb2:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:40:47,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:47,594 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:47,594 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:47,594 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 9a822f4697b07ce815f004ac9ea14bb2/A is initiating minor compaction (all files) 2024-11-21T03:40:47,594 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 9a822f4697b07ce815f004ac9ea14bb2/B is initiating minor compaction (all files) 2024-11-21T03:40:47,594 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a822f4697b07ce815f004ac9ea14bb2/A in TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:47,594 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a822f4697b07ce815f004ac9ea14bb2/B in TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:47,594 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/ff04b1f700c94b1d890a7218ff432c42, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/cf4f1d9268994c798c00f5b0ac111781, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/64dc95e6cc9f460583ecda9b591ee16b] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp, totalSize=36.6 K 2024-11-21T03:40:47,594 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/bdecaedec4ed4590998e18b9522503f1, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/1ce59c7341054d338d8a7c01a1abb4fe, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/c28b2b01720e4e98bbd8f462f0fe0871] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp, totalSize=100.7 K 2024-11-21T03:40:47,595 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:47,595 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. files: [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/bdecaedec4ed4590998e18b9522503f1, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/1ce59c7341054d338d8a7c01a1abb4fe, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/c28b2b01720e4e98bbd8f462f0fe0871] 2024-11-21T03:40:47,595 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdecaedec4ed4590998e18b9522503f1, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732160443742 2024-11-21T03:40:47,595 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting ff04b1f700c94b1d890a7218ff432c42, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732160443742 2024-11-21T03:40:47,595 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ce59c7341054d338d8a7c01a1abb4fe, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732160444990 2024-11-21T03:40:47,595 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting cf4f1d9268994c798c00f5b0ac111781, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732160444990 2024-11-21T03:40:47,596 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 64dc95e6cc9f460583ecda9b591ee16b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732160445948 2024-11-21T03:40:47,596 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting c28b2b01720e4e98bbd8f462f0fe0871, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732160445925 2024-11-21T03:40:47,602 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:47,608 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a822f4697b07ce815f004ac9ea14bb2#B#compaction#181 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:47,609 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/99b40c11c87d41df8976d7d0a2829300 is 50, key is test_row_0/B:col10/1732160445952/Put/seqid=0 2024-11-21T03:40:47,611 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112111486e0e4fe745169b2bd147f9559b8a_9a822f4697b07ce815f004ac9ea14bb2 store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:47,612 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112111486e0e4fe745169b2bd147f9559b8a_9a822f4697b07ce815f004ac9ea14bb2, store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:47,613 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112111486e0e4fe745169b2bd147f9559b8a_9a822f4697b07ce815f004ac9ea14bb2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:47,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742041_1217 (size=13017) 2024-11-21T03:40:47,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742042_1218 (size=4469) 2024-11-21T03:40:47,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-21T03:40:47,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:47,718 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 9a822f4697b07ce815f004ac9ea14bb2 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T03:40:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T03:40:47,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112113b9597769fa45a9a60cbceef012094e_9a822f4697b07ce815f004ac9ea14bb2 is 50, key is test_row_0/A:col10/1732160446628/Put/seqid=0 2024-11-21T03:40:47,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742043_1219 (size=12454) 2024-11-21T03:40:47,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:47,746 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112113b9597769fa45a9a60cbceef012094e_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112113b9597769fa45a9a60cbceef012094e_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:47,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/754807a7b056429c9aa08e2d54d8c705, store: [table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:47,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/754807a7b056429c9aa08e2d54d8c705 is 175, key is test_row_0/A:col10/1732160446628/Put/seqid=0 2024-11-21T03:40:47,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742044_1220 (size=31255) 2024-11-21T03:40:47,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:47,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:47,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160507845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:47,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:47,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:47,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:47,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:47,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160507849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:47,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:47,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:47,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:47,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160507849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:47,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:47,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:47,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:47,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160507957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:47,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:47,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160507957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:47,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:47,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:47,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160507957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:47,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:48,040 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/99b40c11c87d41df8976d7d0a2829300 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/99b40c11c87d41df8976d7d0a2829300 2024-11-21T03:40:48,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T03:40:48,052 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a822f4697b07ce815f004ac9ea14bb2#A#compaction#180 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:48,053 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/5838c0e8fa2140a8a2df73157b5c8d39 is 175, key is test_row_0/A:col10/1732160445952/Put/seqid=0 2024-11-21T03:40:48,054 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a822f4697b07ce815f004ac9ea14bb2/B of 9a822f4697b07ce815f004ac9ea14bb2 into 99b40c11c87d41df8976d7d0a2829300(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:48,054 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:48,054 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., storeName=9a822f4697b07ce815f004ac9ea14bb2/B, priority=13, startTime=1732160447593; duration=0sec 2024-11-21T03:40:48,054 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:48,054 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a822f4697b07ce815f004ac9ea14bb2:B 2024-11-21T03:40:48,054 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:48,055 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:48,055 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 9a822f4697b07ce815f004ac9ea14bb2/C is initiating minor compaction (all files) 2024-11-21T03:40:48,055 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a822f4697b07ce815f004ac9ea14bb2/C in TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:48,055 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/b59e09dd28604825803e990aed41ad33, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/73e3beed7b904a0bade2fc4e4d69a511, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/a0b7015297df4f678197324511758a25] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp, totalSize=36.6 K 2024-11-21T03:40:48,056 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting b59e09dd28604825803e990aed41ad33, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732160443742 2024-11-21T03:40:48,056 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 73e3beed7b904a0bade2fc4e4d69a511, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732160444990 2024-11-21T03:40:48,057 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting a0b7015297df4f678197324511758a25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732160445948 2024-11-21T03:40:48,069 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a822f4697b07ce815f004ac9ea14bb2#C#compaction#183 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:48,070 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/222510b44acc4f9ebea83207bedba625 is 50, key is test_row_0/C:col10/1732160445952/Put/seqid=0 2024-11-21T03:40:48,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742046_1222 (size=13017) 2024-11-21T03:40:48,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742045_1221 (size=31971) 2024-11-21T03:40:48,156 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=325, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/754807a7b056429c9aa08e2d54d8c705 2024-11-21T03:40:48,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/81d7dc7d1b61499492a7d4ec954be8f4 is 50, key is test_row_0/B:col10/1732160446628/Put/seqid=0 2024-11-21T03:40:48,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:48,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160508167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:48,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:48,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:48,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160508167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:48,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:48,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:48,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:48,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:48,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:48,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:48,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160508167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:48,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:48,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:48,171 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:48,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742047_1223 (size=12301) 2024-11-21T03:40:48,176 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/81d7dc7d1b61499492a7d4ec954be8f4 2024-11-21T03:40:48,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/fa64d1e1a4f34cc9b7a67fc17aaab849 is 50, key is test_row_0/C:col10/1732160446628/Put/seqid=0 2024-11-21T03:40:48,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742048_1224 (size=12301) 2024-11-21T03:40:48,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160508476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:48,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:48,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160508476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:48,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:48,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160508477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:48,501 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/222510b44acc4f9ebea83207bedba625 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/222510b44acc4f9ebea83207bedba625 2024-11-21T03:40:48,502 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/5838c0e8fa2140a8a2df73157b5c8d39 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/5838c0e8fa2140a8a2df73157b5c8d39 2024-11-21T03:40:48,508 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a822f4697b07ce815f004ac9ea14bb2/C of 9a822f4697b07ce815f004ac9ea14bb2 into 222510b44acc4f9ebea83207bedba625(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:48,509 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:48,509 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., storeName=9a822f4697b07ce815f004ac9ea14bb2/C, priority=13, startTime=1732160447593; duration=0sec 2024-11-21T03:40:48,509 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:48,509 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a822f4697b07ce815f004ac9ea14bb2:C 2024-11-21T03:40:48,509 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a822f4697b07ce815f004ac9ea14bb2/A of 9a822f4697b07ce815f004ac9ea14bb2 into 5838c0e8fa2140a8a2df73157b5c8d39(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:48,509 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:48,509 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., storeName=9a822f4697b07ce815f004ac9ea14bb2/A, priority=13, startTime=1732160447593; duration=0sec 2024-11-21T03:40:48,509 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:48,509 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a822f4697b07ce815f004ac9ea14bb2:A 2024-11-21T03:40:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T03:40:48,594 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/fa64d1e1a4f34cc9b7a67fc17aaab849 2024-11-21T03:40:48,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/754807a7b056429c9aa08e2d54d8c705 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/754807a7b056429c9aa08e2d54d8c705 2024-11-21T03:40:48,603 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/754807a7b056429c9aa08e2d54d8c705, entries=150, sequenceid=325, filesize=30.5 K 2024-11-21T03:40:48,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/81d7dc7d1b61499492a7d4ec954be8f4 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/81d7dc7d1b61499492a7d4ec954be8f4 2024-11-21T03:40:48,609 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/81d7dc7d1b61499492a7d4ec954be8f4, entries=150, sequenceid=325, filesize=12.0 K 2024-11-21T03:40:48,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/fa64d1e1a4f34cc9b7a67fc17aaab849 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/fa64d1e1a4f34cc9b7a67fc17aaab849 2024-11-21T03:40:48,614 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/fa64d1e1a4f34cc9b7a67fc17aaab849, entries=150, sequenceid=325, filesize=12.0 K 2024-11-21T03:40:48,615 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 9a822f4697b07ce815f004ac9ea14bb2 in 897ms, sequenceid=325, compaction requested=false 2024-11-21T03:40:48,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:48,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:48,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-21T03:40:48,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-21T03:40:48,618 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-21T03:40:48,619 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2120 sec 2024-11-21T03:40:48,620 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.2190 sec 2024-11-21T03:40:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:48,989 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:48,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a822f4697b07ce815f004ac9ea14bb2 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T03:40:48,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121816cd966b61640af859b4fb014b9c566_9a822f4697b07ce815f004ac9ea14bb2 is 50, key is test_row_0/A:col10/1732160447846/Put/seqid=0 2024-11-21T03:40:49,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742049_1225 (size=14994) 2024-11-21T03:40:49,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160509018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:49,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160509019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:49,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:49,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160509024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:49,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:49,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160509127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:49,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:49,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160509127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:49,130 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:49,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:49,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:49,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160509136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:49,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:49,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:49,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37578 deadline: 1732160509197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:49,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:49,200 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_2' in 9a822f4697b07ce815f004ac9ea14bb2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:40:49,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37568 deadline: 1732160509197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:49,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:49,201 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_2' in 9a822f4697b07ce815f004ac9ea14bb2 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:40:49,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:49,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:49,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160509337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:49,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:49,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:49,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160509337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:49,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:49,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:49,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160509346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:49,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:49,401 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:49,405 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121816cd966b61640af859b4fb014b9c566_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121816cd966b61640af859b4fb014b9c566_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:49,406 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/c5eb5c5c0f864c3993268fb32f7f0d2d, store: [table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:49,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/c5eb5c5c0f864c3993268fb32f7f0d2d is 175, key is test_row_0/A:col10/1732160447846/Put/seqid=0 2024-11-21T03:40:49,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742050_1226 (size=39949) 2024-11-21T03:40:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-21T03:40:49,557 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:49,558 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:40:49,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-21T03:40:49,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T03:40:49,559 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:40:49,560 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:40:49,560 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:40:49,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160509647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160509647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:49,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160509657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T03:40:49,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-21T03:40:49,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:49,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:49,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:49,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:49,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:49,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:49,810 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/c5eb5c5c0f864c3993268fb32f7f0d2d 2024-11-21T03:40:49,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/c50db14ca2e54faaa498b2b0a78ce8d5 is 50, key is test_row_0/B:col10/1732160447846/Put/seqid=0 2024-11-21T03:40:49,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742051_1227 (size=12301) 2024-11-21T03:40:49,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-21T03:40:49,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:49,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:49,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:49,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:49,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:49,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T03:40:50,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-21T03:40:50,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:50,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:50,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:50,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:50,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:50,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:50,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:50,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160510157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:50,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:50,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160510157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:50,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-21T03:40:50,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:50,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:50,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:50,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:50,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:50,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:50,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160510167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:50,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:50,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:50,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:50,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:50,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T03:40:50,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/c50db14ca2e54faaa498b2b0a78ce8d5 2024-11-21T03:40:50,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/4cedab1c18c5431d8835ea6d430019d9 is 50, key is test_row_0/C:col10/1732160447846/Put/seqid=0 2024-11-21T03:40:50,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742052_1228 (size=12301) 2024-11-21T03:40:50,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/4cedab1c18c5431d8835ea6d430019d9 2024-11-21T03:40:50,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/c5eb5c5c0f864c3993268fb32f7f0d2d as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/c5eb5c5c0f864c3993268fb32f7f0d2d 2024-11-21T03:40:50,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/c5eb5c5c0f864c3993268fb32f7f0d2d, entries=200, sequenceid=342, filesize=39.0 K 2024-11-21T03:40:50,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/c50db14ca2e54faaa498b2b0a78ce8d5 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/c50db14ca2e54faaa498b2b0a78ce8d5 2024-11-21T03:40:50,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/c50db14ca2e54faaa498b2b0a78ce8d5, entries=150, sequenceid=342, filesize=12.0 K 2024-11-21T03:40:50,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/4cedab1c18c5431d8835ea6d430019d9 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/4cedab1c18c5431d8835ea6d430019d9 2024-11-21T03:40:50,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/4cedab1c18c5431d8835ea6d430019d9, entries=150, sequenceid=342, filesize=12.0 K 2024-11-21T03:40:50,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 9a822f4697b07ce815f004ac9ea14bb2 in 1265ms, sequenceid=342, compaction requested=true 2024-11-21T03:40:50,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:50,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a822f4697b07ce815f004ac9ea14bb2:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:40:50,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:50,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a822f4697b07ce815f004ac9ea14bb2:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:40:50,254 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:50,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:50,254 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:50,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a822f4697b07ce815f004ac9ea14bb2:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:40:50,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:50,255 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:50,255 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 9a822f4697b07ce815f004ac9ea14bb2/B is initiating minor compaction (all files) 2024-11-21T03:40:50,255 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a822f4697b07ce815f004ac9ea14bb2/B in TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:50,255 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/99b40c11c87d41df8976d7d0a2829300, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/81d7dc7d1b61499492a7d4ec954be8f4, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/c50db14ca2e54faaa498b2b0a78ce8d5] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp, totalSize=36.7 K 2024-11-21T03:40:50,257 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:50,257 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 99b40c11c87d41df8976d7d0a2829300, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732160445948 2024-11-21T03:40:50,257 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 9a822f4697b07ce815f004ac9ea14bb2/A is initiating minor compaction (all files) 2024-11-21T03:40:50,257 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a822f4697b07ce815f004ac9ea14bb2/A in TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:50,257 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/5838c0e8fa2140a8a2df73157b5c8d39, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/754807a7b056429c9aa08e2d54d8c705, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/c5eb5c5c0f864c3993268fb32f7f0d2d] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp, totalSize=100.8 K 2024-11-21T03:40:50,257 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 81d7dc7d1b61499492a7d4ec954be8f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732160446626 2024-11-21T03:40:50,257 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:50,257 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. files: [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/5838c0e8fa2140a8a2df73157b5c8d39, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/754807a7b056429c9aa08e2d54d8c705, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/c5eb5c5c0f864c3993268fb32f7f0d2d] 2024-11-21T03:40:50,257 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting c50db14ca2e54faaa498b2b0a78ce8d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732160447837 2024-11-21T03:40:50,257 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5838c0e8fa2140a8a2df73157b5c8d39, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732160445948 2024-11-21T03:40:50,258 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 754807a7b056429c9aa08e2d54d8c705, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732160446626 2024-11-21T03:40:50,258 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5eb5c5c0f864c3993268fb32f7f0d2d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732160447837 2024-11-21T03:40:50,265 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:50,265 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a822f4697b07ce815f004ac9ea14bb2#B#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:50,266 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/cd57bf153f0a4fdd9ebd1d1f96d1d5c9 is 50, key is test_row_0/B:col10/1732160447846/Put/seqid=0 2024-11-21T03:40:50,271 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112106122b701ab34816a1f98115965827f8_9a822f4697b07ce815f004ac9ea14bb2 store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:50,273 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112106122b701ab34816a1f98115965827f8_9a822f4697b07ce815f004ac9ea14bb2, store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:50,273 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112106122b701ab34816a1f98115965827f8_9a822f4697b07ce815f004ac9ea14bb2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:50,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742053_1229 (size=13119) 2024-11-21T03:40:50,286 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/cd57bf153f0a4fdd9ebd1d1f96d1d5c9 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/cd57bf153f0a4fdd9ebd1d1f96d1d5c9 2024-11-21T03:40:50,293 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a822f4697b07ce815f004ac9ea14bb2/B of 9a822f4697b07ce815f004ac9ea14bb2 into cd57bf153f0a4fdd9ebd1d1f96d1d5c9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:50,293 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:50,293 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., storeName=9a822f4697b07ce815f004ac9ea14bb2/B, priority=13, startTime=1732160450254; duration=0sec 2024-11-21T03:40:50,293 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:50,293 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a822f4697b07ce815f004ac9ea14bb2:B 2024-11-21T03:40:50,293 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:50,294 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:50,294 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 9a822f4697b07ce815f004ac9ea14bb2/C is initiating minor compaction (all files) 2024-11-21T03:40:50,294 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a822f4697b07ce815f004ac9ea14bb2/C in TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:50,294 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/222510b44acc4f9ebea83207bedba625, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/fa64d1e1a4f34cc9b7a67fc17aaab849, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/4cedab1c18c5431d8835ea6d430019d9] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp, totalSize=36.7 K 2024-11-21T03:40:50,294 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 222510b44acc4f9ebea83207bedba625, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1732160445948 2024-11-21T03:40:50,295 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting fa64d1e1a4f34cc9b7a67fc17aaab849, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732160446626 2024-11-21T03:40:50,295 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cedab1c18c5431d8835ea6d430019d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732160447837 2024-11-21T03:40:50,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742054_1230 (size=4469) 2024-11-21T03:40:50,302 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a822f4697b07ce815f004ac9ea14bb2#A#compaction#190 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:50,302 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/ad87dac2160048719ba01b0a6be43b67 is 175, key is test_row_0/A:col10/1732160447846/Put/seqid=0 2024-11-21T03:40:50,307 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a822f4697b07ce815f004ac9ea14bb2#C#compaction#191 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:50,308 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/546f41f84cfc4c97af73a4553a478854 is 50, key is test_row_0/C:col10/1732160447846/Put/seqid=0 2024-11-21T03:40:50,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-21T03:40:50,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:50,323 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 9a822f4697b07ce815f004ac9ea14bb2 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-21T03:40:50,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742055_1231 (size=32073) 2024-11-21T03:40:50,334 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/ad87dac2160048719ba01b0a6be43b67 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/ad87dac2160048719ba01b0a6be43b67 2024-11-21T03:40:50,339 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a822f4697b07ce815f004ac9ea14bb2/A of 9a822f4697b07ce815f004ac9ea14bb2 into ad87dac2160048719ba01b0a6be43b67(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:50,339 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:50,339 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., storeName=9a822f4697b07ce815f004ac9ea14bb2/A, priority=13, startTime=1732160450254; duration=0sec 2024-11-21T03:40:50,339 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:50,339 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a822f4697b07ce815f004ac9ea14bb2:A 2024-11-21T03:40:50,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742056_1232 (size=13119) 2024-11-21T03:40:50,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219e9779473c0f4f31ba72a14c63b99d42_9a822f4697b07ce815f004ac9ea14bb2 is 50, key is test_row_0/A:col10/1732160449017/Put/seqid=0 2024-11-21T03:40:50,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742057_1233 (size=12454) 2024-11-21T03:40:50,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T03:40:50,759 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/546f41f84cfc4c97af73a4553a478854 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/546f41f84cfc4c97af73a4553a478854 2024-11-21T03:40:50,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:50,765 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a822f4697b07ce815f004ac9ea14bb2/C of 9a822f4697b07ce815f004ac9ea14bb2 into 546f41f84cfc4c97af73a4553a478854(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:50,765 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:50,765 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., storeName=9a822f4697b07ce815f004ac9ea14bb2/C, priority=13, startTime=1732160450254; duration=0sec 2024-11-21T03:40:50,765 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:50,765 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a822f4697b07ce815f004ac9ea14bb2:C 2024-11-21T03:40:50,768 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219e9779473c0f4f31ba72a14c63b99d42_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219e9779473c0f4f31ba72a14c63b99d42_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:50,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/70f0d6c01a9c479f8339e117a33c3194, store: [table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:50,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/70f0d6c01a9c479f8339e117a33c3194 is 175, key is test_row_0/A:col10/1732160449017/Put/seqid=0 2024-11-21T03:40:50,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742058_1234 (size=31255) 2024-11-21T03:40:50,787 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=365, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/70f0d6c01a9c479f8339e117a33c3194 2024-11-21T03:40:50,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/4034044f4a5f429086ad36e32e6a270e is 50, key is test_row_0/B:col10/1732160449017/Put/seqid=0 2024-11-21T03:40:50,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742059_1235 (size=12301) 2024-11-21T03:40:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:51,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. as already flushing 2024-11-21T03:40:51,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160511183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:51,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:51,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:51,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:51,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160511184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:51,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:51,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37588 deadline: 1732160511187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:51,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:51,188 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:51,188 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_0' in 9a822f4697b07ce815f004ac9ea14bb2 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:40:51,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:51,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:51,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:51,211 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/4034044f4a5f429086ad36e32e6a270e 2024-11-21T03:40:51,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/b2e518d05c00462496e9405071ca0826 is 50, key is test_row_0/C:col10/1732160449017/Put/seqid=0 2024-11-21T03:40:51,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742060_1236 (size=12301) 2024-11-21T03:40:51,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160511297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:51,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:51,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:51,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:51,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160511297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:51,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:51,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:51,299 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:51,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:51,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37562 deadline: 1732160511506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37514 deadline: 1732160511506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:40:51,614 INFO [Thread-670 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-670. 2024-11-21T03:40:51,614 INFO [Thread-672 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-672. 2024-11-21T03:40:51,614 DEBUG [Thread-670 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:40:51,614 DEBUG [Thread-670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:51,614 DEBUG [Thread-672 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:40:51,614 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:51,614 DEBUG [Thread-672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:51,614 DEBUG [Thread-670 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:51,614 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:40:51,614 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:40:51,620 INFO [Thread-676 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-676. 2024-11-21T03:40:51,620 DEBUG [Thread-676 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:40:51,620 DEBUG [Thread-676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:51,620 DEBUG [Thread-676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:51,621 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:40:51,622 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/b2e518d05c00462496e9405071ca0826 2024-11-21T03:40:51,623 INFO [Thread-674 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-674. 2024-11-21T03:40:51,623 DEBUG [Thread-674 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:40:51,623 DEBUG [Thread-674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:51,624 DEBUG [Thread-674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:51,624 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:40:51,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/70f0d6c01a9c479f8339e117a33c3194 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/70f0d6c01a9c479f8339e117a33c3194 2024-11-21T03:40:51,630 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/70f0d6c01a9c479f8339e117a33c3194, entries=150, sequenceid=365, filesize=30.5 K 2024-11-21T03:40:51,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/4034044f4a5f429086ad36e32e6a270e as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/4034044f4a5f429086ad36e32e6a270e 2024-11-21T03:40:51,679 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/4034044f4a5f429086ad36e32e6a270e, entries=150, sequenceid=365, filesize=12.0 K 2024-11-21T03:40:51,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/b2e518d05c00462496e9405071ca0826 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/b2e518d05c00462496e9405071ca0826 2024-11-21T03:40:51,688 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/b2e518d05c00462496e9405071ca0826, entries=150, sequenceid=365, filesize=12.0 K 2024-11-21T03:40:51,688 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 9a822f4697b07ce815f004ac9ea14bb2 in 1365ms, sequenceid=365, compaction requested=false 2024-11-21T03:40:51,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:51,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:51,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-21T03:40:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-21T03:40:51,691 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-21T03:40:51,691 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1300 sec 2024-11-21T03:40:51,693 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 2.1340 sec 2024-11-21T03:40:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T03:40:51,707 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:51,819 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:51,820 INFO [Thread-663 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-663. 2024-11-21T03:40:51,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 9a822f4697b07ce815f004ac9ea14bb2 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-21T03:40:51,820 DEBUG [Thread-663 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:40:51,820 DEBUG [Thread-663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:51,820 DEBUG [Thread-663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:51,820 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:40:51,820 INFO [Thread-659 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-659. 2024-11-21T03:40:51,820 DEBUG [Thread-659 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:40:51,820 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:51,821 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:51,821 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:40:51,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121719d543f28d144a58efd4f4a7711f2f8_9a822f4697b07ce815f004ac9ea14bb2 is 50, key is test_row_0/A:col10/1732160451817/Put/seqid=0 2024-11-21T03:40:51,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742061_1237 (size=12454) 2024-11-21T03:40:52,235 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:52,240 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121719d543f28d144a58efd4f4a7711f2f8_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121719d543f28d144a58efd4f4a7711f2f8_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:52,241 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/e82f30452e20479bab6176631f68176c, store: [table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:52,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/e82f30452e20479bab6176631f68176c is 175, key is test_row_0/A:col10/1732160451817/Put/seqid=0 2024-11-21T03:40:52,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742062_1238 (size=31255) 2024-11-21T03:40:52,646 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=383, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/e82f30452e20479bab6176631f68176c 2024-11-21T03:40:52,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/0189d0c32eba464496c161242815e438 is 50, key is test_row_0/B:col10/1732160451817/Put/seqid=0 2024-11-21T03:40:52,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742063_1239 (size=12301) 2024-11-21T03:40:53,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/0189d0c32eba464496c161242815e438 2024-11-21T03:40:53,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/3cdcd27719aa46fe9ff4340814031c59 is 50, key is test_row_0/C:col10/1732160451817/Put/seqid=0 2024-11-21T03:40:53,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742064_1240 (size=12301) 2024-11-21T03:40:53,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/3cdcd27719aa46fe9ff4340814031c59 2024-11-21T03:40:53,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/e82f30452e20479bab6176631f68176c as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/e82f30452e20479bab6176631f68176c 2024-11-21T03:40:53,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/e82f30452e20479bab6176631f68176c, entries=150, sequenceid=383, filesize=30.5 K 2024-11-21T03:40:53,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/0189d0c32eba464496c161242815e438 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/0189d0c32eba464496c161242815e438 2024-11-21T03:40:53,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/0189d0c32eba464496c161242815e438, entries=150, sequenceid=383, filesize=12.0 K 2024-11-21T03:40:53,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/3cdcd27719aa46fe9ff4340814031c59 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/3cdcd27719aa46fe9ff4340814031c59 2024-11-21T03:40:53,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/3cdcd27719aa46fe9ff4340814031c59, entries=150, sequenceid=383, filesize=12.0 K 2024-11-21T03:40:53,101 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=0 B/0 for 9a822f4697b07ce815f004ac9ea14bb2 in 1282ms, sequenceid=383, compaction requested=true 2024-11-21T03:40:53,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:53,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a822f4697b07ce815f004ac9ea14bb2:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:40:53,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:53,102 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:53,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a822f4697b07ce815f004ac9ea14bb2:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:40:53,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:53,102 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:53,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9a822f4697b07ce815f004ac9ea14bb2:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:40:53,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:53,103 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94583 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:53,103 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:53,103 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 9a822f4697b07ce815f004ac9ea14bb2/A is initiating minor compaction (all files) 2024-11-21T03:40:53,103 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 9a822f4697b07ce815f004ac9ea14bb2/B is initiating minor compaction (all files) 2024-11-21T03:40:53,103 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a822f4697b07ce815f004ac9ea14bb2/B in TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:53,103 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a822f4697b07ce815f004ac9ea14bb2/A in TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:53,103 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/ad87dac2160048719ba01b0a6be43b67, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/70f0d6c01a9c479f8339e117a33c3194, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/e82f30452e20479bab6176631f68176c] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp, totalSize=92.4 K 2024-11-21T03:40:53,103 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/cd57bf153f0a4fdd9ebd1d1f96d1d5c9, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/4034044f4a5f429086ad36e32e6a270e, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/0189d0c32eba464496c161242815e438] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp, totalSize=36.8 K 2024-11-21T03:40:53,103 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:53,103 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. files: [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/ad87dac2160048719ba01b0a6be43b67, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/70f0d6c01a9c479f8339e117a33c3194, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/e82f30452e20479bab6176631f68176c] 2024-11-21T03:40:53,103 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting cd57bf153f0a4fdd9ebd1d1f96d1d5c9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732160447837 2024-11-21T03:40:53,104 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad87dac2160048719ba01b0a6be43b67, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732160447837 2024-11-21T03:40:53,104 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 4034044f4a5f429086ad36e32e6a270e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732160449012 2024-11-21T03:40:53,104 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70f0d6c01a9c479f8339e117a33c3194, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732160449012 2024-11-21T03:40:53,104 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting e82f30452e20479bab6176631f68176c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732160451183 2024-11-21T03:40:53,104 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0189d0c32eba464496c161242815e438, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732160451183 2024-11-21T03:40:53,114 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a822f4697b07ce815f004ac9ea14bb2#B#compaction#198 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:53,114 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/7dee3aee203d47598ba5a866597aae46 is 50, key is test_row_0/B:col10/1732160451817/Put/seqid=0 2024-11-21T03:40:53,115 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:53,122 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121bf26f6e44d584fd89dcca5f3132badaf_9a822f4697b07ce815f004ac9ea14bb2 store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:53,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742065_1241 (size=13221) 2024-11-21T03:40:53,148 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121bf26f6e44d584fd89dcca5f3132badaf_9a822f4697b07ce815f004ac9ea14bb2, store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:53,148 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bf26f6e44d584fd89dcca5f3132badaf_9a822f4697b07ce815f004ac9ea14bb2 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:53,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742066_1242 (size=4469) 2024-11-21T03:40:53,158 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a822f4697b07ce815f004ac9ea14bb2#A#compaction#199 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:53,159 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/5c846a2355d2454eba1d5e765ebc99b2 is 175, key is test_row_0/A:col10/1732160451817/Put/seqid=0 2024-11-21T03:40:53,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742067_1243 (size=32175) 2024-11-21T03:40:53,176 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/5c846a2355d2454eba1d5e765ebc99b2 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/5c846a2355d2454eba1d5e765ebc99b2 2024-11-21T03:40:53,183 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a822f4697b07ce815f004ac9ea14bb2/A of 9a822f4697b07ce815f004ac9ea14bb2 into 5c846a2355d2454eba1d5e765ebc99b2(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:53,183 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:53,183 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., storeName=9a822f4697b07ce815f004ac9ea14bb2/A, priority=13, startTime=1732160453102; duration=0sec 2024-11-21T03:40:53,183 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:40:53,183 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a822f4697b07ce815f004ac9ea14bb2:A 2024-11-21T03:40:53,183 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:40:53,184 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:40:53,185 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 9a822f4697b07ce815f004ac9ea14bb2/C is initiating minor compaction (all files) 2024-11-21T03:40:53,185 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 9a822f4697b07ce815f004ac9ea14bb2/C in TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:53,185 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/546f41f84cfc4c97af73a4553a478854, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/b2e518d05c00462496e9405071ca0826, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/3cdcd27719aa46fe9ff4340814031c59] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp, totalSize=36.8 K 2024-11-21T03:40:53,185 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 546f41f84cfc4c97af73a4553a478854, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732160447837 2024-11-21T03:40:53,186 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2e518d05c00462496e9405071ca0826, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1732160449012 2024-11-21T03:40:53,186 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cdcd27719aa46fe9ff4340814031c59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1732160451183 2024-11-21T03:40:53,203 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9a822f4697b07ce815f004ac9ea14bb2#C#compaction#200 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:40:53,204 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/66e6a48a7fa343bf9e752d049799e9b6 is 50, key is test_row_0/C:col10/1732160451817/Put/seqid=0 2024-11-21T03:40:53,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742068_1244 (size=13221) 2024-11-21T03:40:53,212 INFO [Thread-661 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-661. 2024-11-21T03:40:53,212 INFO [Thread-667 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-667. 2024-11-21T03:40:53,212 DEBUG [Thread-661 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:40:53,212 DEBUG [Thread-667 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:40:53,212 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:53,212 DEBUG [Thread-667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:53,212 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:53,212 DEBUG [Thread-667 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:53,212 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:40:53,212 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:40:53,220 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/66e6a48a7fa343bf9e752d049799e9b6 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/66e6a48a7fa343bf9e752d049799e9b6 2024-11-21T03:40:53,224 INFO [Thread-665 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-665. 2024-11-21T03:40:53,224 DEBUG [Thread-665 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:40:53,224 DEBUG [Thread-665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:53,224 DEBUG [Thread-665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:53,224 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:40:53,224 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T03:40:53,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-21T03:40:53,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-21T03:40:53,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-21T03:40:53,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-21T03:40:53,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-21T03:40:53,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T03:40:53,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3356 2024-11-21T03:40:53,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3309 2024-11-21T03:40:53,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T03:40:53,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1482 2024-11-21T03:40:53,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4442 rows 2024-11-21T03:40:53,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1492 2024-11-21T03:40:53,225 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4472 rows 2024-11-21T03:40:53,225 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T03:40:53,225 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T03:40:53,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:53,225 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:53,225 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:40:53,227 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a822f4697b07ce815f004ac9ea14bb2/C of 9a822f4697b07ce815f004ac9ea14bb2 into 66e6a48a7fa343bf9e752d049799e9b6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:53,227 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:53,227 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., storeName=9a822f4697b07ce815f004ac9ea14bb2/C, priority=13, startTime=1732160453102; duration=0sec 2024-11-21T03:40:53,227 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T03:40:53,227 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:53,227 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a822f4697b07ce815f004ac9ea14bb2:C 2024-11-21T03:40:53,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T03:40:53,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-21T03:40:53,232 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732160453232"}]},"ts":"1732160453232"} 2024-11-21T03:40:53,234 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T03:40:53,234 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T03:40:53,235 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T03:40:53,236 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a822f4697b07ce815f004ac9ea14bb2, UNASSIGN}] 2024-11-21T03:40:53,238 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a822f4697b07ce815f004ac9ea14bb2, UNASSIGN 2024-11-21T03:40:53,239 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=9a822f4697b07ce815f004ac9ea14bb2, regionState=CLOSING, regionLocation=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:53,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a822f4697b07ce815f004ac9ea14bb2, UNASSIGN because future has completed 2024-11-21T03:40:53,251 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T03:40:53,251 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889}] 2024-11-21T03:40:53,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-21T03:40:53,404 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(122): Close 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:53,404 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T03:40:53,404 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1722): Closing 9a822f4697b07ce815f004ac9ea14bb2, disabling compactions & flushes 2024-11-21T03:40:53,404 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1993): waiting for 1 compactions to complete for region TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:53,532 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/7dee3aee203d47598ba5a866597aae46 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/7dee3aee203d47598ba5a866597aae46 2024-11-21T03:40:53,540 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 9a822f4697b07ce815f004ac9ea14bb2/B of 9a822f4697b07ce815f004ac9ea14bb2 into 7dee3aee203d47598ba5a866597aae46(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:40:53,540 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:53,540 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:53,540 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. after waiting 0 ms 2024-11-21T03:40:53,540 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:53,540 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(2902): Flushing 9a822f4697b07ce815f004ac9ea14bb2 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-21T03:40:53,541 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 9a822f4697b07ce815f004ac9ea14bb2: 2024-11-21T03:40:53,541 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., storeName=9a822f4697b07ce815f004ac9ea14bb2/B, priority=13, startTime=1732160453102; duration=0sec 2024-11-21T03:40:53,541 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:40:53,541 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9a822f4697b07ce815f004ac9ea14bb2:B 2024-11-21T03:40:53,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-21T03:40:53,551 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a39ce109b780465c8146760557759e94_9a822f4697b07ce815f004ac9ea14bb2 is 50, key is test_row_0/A:col10/1732160453207/Put/seqid=0 2024-11-21T03:40:53,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742069_1245 (size=9914) 2024-11-21T03:40:53,561 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:53,565 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a39ce109b780465c8146760557759e94_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a39ce109b780465c8146760557759e94_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:53,566 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/88947262ac5d46c5824be48eb9ddbc7d, store: [table=TestAcidGuarantees family=A region=9a822f4697b07ce815f004ac9ea14bb2] 2024-11-21T03:40:53,567 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/88947262ac5d46c5824be48eb9ddbc7d is 175, key is test_row_0/A:col10/1732160453207/Put/seqid=0 2024-11-21T03:40:53,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742070_1246 (size=22561) 2024-11-21T03:40:53,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-21T03:40:53,972 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=392, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/88947262ac5d46c5824be48eb9ddbc7d 2024-11-21T03:40:53,979 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/5921565f533044d5bdc6a3a209726bce is 50, key is test_row_0/B:col10/1732160453207/Put/seqid=0 2024-11-21T03:40:53,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742071_1247 (size=9857) 2024-11-21T03:40:54,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-21T03:40:54,384 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/5921565f533044d5bdc6a3a209726bce 2024-11-21T03:40:54,390 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/869e9ca2c91043e2b48649de82edbf39 is 50, key is test_row_0/C:col10/1732160453207/Put/seqid=0 2024-11-21T03:40:54,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742072_1248 (size=9857) 2024-11-21T03:40:54,794 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/869e9ca2c91043e2b48649de82edbf39 2024-11-21T03:40:54,800 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/A/88947262ac5d46c5824be48eb9ddbc7d as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/88947262ac5d46c5824be48eb9ddbc7d 2024-11-21T03:40:54,804 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/88947262ac5d46c5824be48eb9ddbc7d, entries=100, sequenceid=392, filesize=22.0 K 2024-11-21T03:40:54,805 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/B/5921565f533044d5bdc6a3a209726bce as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/5921565f533044d5bdc6a3a209726bce 2024-11-21T03:40:54,810 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/5921565f533044d5bdc6a3a209726bce, entries=100, sequenceid=392, filesize=9.6 K 2024-11-21T03:40:54,811 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/.tmp/C/869e9ca2c91043e2b48649de82edbf39 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/869e9ca2c91043e2b48649de82edbf39 2024-11-21T03:40:54,815 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/869e9ca2c91043e2b48649de82edbf39, entries=100, sequenceid=392, filesize=9.6 K 2024-11-21T03:40:54,815 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 9a822f4697b07ce815f004ac9ea14bb2 in 1275ms, sequenceid=392, compaction requested=false 2024-11-21T03:40:54,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/7680e8e727104082b4d12bd395b20f42, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/cef507059fa84b49a112fb85428f140b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/64b136fa72c94b2a8c6afe8cae971488, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/0f017242894b4926a6e040432fd88d17, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/0add856224c14382844ea10e8a3945d0, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/87ca946911b748afbafcfce3f47f48e3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/b01c10ac959e4413bad147aa5a293662, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/f66f90cdc25c40d4a017e7a1b3178fb5, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/305ccb6b5e114c21a96b29e15013416b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/2d47c0b5d4dd43ebaa5c9825740efc95, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/d176cba569bd4f71b19c71ce95c8d589, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/5c48457436a94fad9fa45ba16e68ff4d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/477547b4a3064de5886bbc63ef39691c, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/c122007c522e49f18f594c218b7c8601, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/7b48c86e313c44c99a7856986576695e, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/86ff4be961dd420e9f36fb99a492460e, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/f6ec49fc7a104ff89db5b6ccee8b7f35, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/5d87d39a7b1c46029ea92ed14ebd16b3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/bdecaedec4ed4590998e18b9522503f1, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/1ce59c7341054d338d8a7c01a1abb4fe, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/c28b2b01720e4e98bbd8f462f0fe0871, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/5838c0e8fa2140a8a2df73157b5c8d39, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/754807a7b056429c9aa08e2d54d8c705, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/c5eb5c5c0f864c3993268fb32f7f0d2d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/ad87dac2160048719ba01b0a6be43b67, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/70f0d6c01a9c479f8339e117a33c3194, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/e82f30452e20479bab6176631f68176c] to archive 2024-11-21T03:40:54,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:40:54,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/7680e8e727104082b4d12bd395b20f42 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/7680e8e727104082b4d12bd395b20f42 2024-11-21T03:40:54,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/cef507059fa84b49a112fb85428f140b to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/cef507059fa84b49a112fb85428f140b 2024-11-21T03:40:54,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/64b136fa72c94b2a8c6afe8cae971488 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/64b136fa72c94b2a8c6afe8cae971488 2024-11-21T03:40:54,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/0f017242894b4926a6e040432fd88d17 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/0f017242894b4926a6e040432fd88d17 2024-11-21T03:40:54,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/0add856224c14382844ea10e8a3945d0 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/0add856224c14382844ea10e8a3945d0 2024-11-21T03:40:54,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/87ca946911b748afbafcfce3f47f48e3 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/87ca946911b748afbafcfce3f47f48e3 2024-11-21T03:40:54,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/b01c10ac959e4413bad147aa5a293662 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/b01c10ac959e4413bad147aa5a293662 2024-11-21T03:40:54,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/f66f90cdc25c40d4a017e7a1b3178fb5 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/f66f90cdc25c40d4a017e7a1b3178fb5 2024-11-21T03:40:54,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/305ccb6b5e114c21a96b29e15013416b to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/305ccb6b5e114c21a96b29e15013416b 2024-11-21T03:40:54,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/2d47c0b5d4dd43ebaa5c9825740efc95 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/2d47c0b5d4dd43ebaa5c9825740efc95 2024-11-21T03:40:54,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/d176cba569bd4f71b19c71ce95c8d589 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/d176cba569bd4f71b19c71ce95c8d589 2024-11-21T03:40:54,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/5c48457436a94fad9fa45ba16e68ff4d to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/5c48457436a94fad9fa45ba16e68ff4d 2024-11-21T03:40:54,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/477547b4a3064de5886bbc63ef39691c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/477547b4a3064de5886bbc63ef39691c 2024-11-21T03:40:54,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/c122007c522e49f18f594c218b7c8601 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/c122007c522e49f18f594c218b7c8601 2024-11-21T03:40:54,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/7b48c86e313c44c99a7856986576695e to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/7b48c86e313c44c99a7856986576695e 2024-11-21T03:40:54,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/86ff4be961dd420e9f36fb99a492460e to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/86ff4be961dd420e9f36fb99a492460e 2024-11-21T03:40:54,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/f6ec49fc7a104ff89db5b6ccee8b7f35 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/f6ec49fc7a104ff89db5b6ccee8b7f35 2024-11-21T03:40:54,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/5d87d39a7b1c46029ea92ed14ebd16b3 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/5d87d39a7b1c46029ea92ed14ebd16b3 2024-11-21T03:40:54,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/bdecaedec4ed4590998e18b9522503f1 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/bdecaedec4ed4590998e18b9522503f1 2024-11-21T03:40:54,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/1ce59c7341054d338d8a7c01a1abb4fe to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/1ce59c7341054d338d8a7c01a1abb4fe 2024-11-21T03:40:54,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/c28b2b01720e4e98bbd8f462f0fe0871 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/c28b2b01720e4e98bbd8f462f0fe0871 2024-11-21T03:40:54,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/5838c0e8fa2140a8a2df73157b5c8d39 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/5838c0e8fa2140a8a2df73157b5c8d39 2024-11-21T03:40:54,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/754807a7b056429c9aa08e2d54d8c705 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/754807a7b056429c9aa08e2d54d8c705 2024-11-21T03:40:54,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/c5eb5c5c0f864c3993268fb32f7f0d2d to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/c5eb5c5c0f864c3993268fb32f7f0d2d 2024-11-21T03:40:54,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/ad87dac2160048719ba01b0a6be43b67 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/ad87dac2160048719ba01b0a6be43b67 2024-11-21T03:40:54,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/70f0d6c01a9c479f8339e117a33c3194 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/70f0d6c01a9c479f8339e117a33c3194 2024-11-21T03:40:54,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/e82f30452e20479bab6176631f68176c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/e82f30452e20479bab6176631f68176c 2024-11-21T03:40:54,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/43dffd27eda04baf8b20f32c377438fc, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/a19061041a9a46a9abec7d873814b540, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/a0fd47f2ad384a7d8a2061aac5e9c3b7, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/83dfbaa3d10c447ca9400d38aedaae7b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/a8f078debfe74eeb9549f8ac4f882428, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/9edddfaeee374d2689ca2596fb74495f, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/a0fb333243a24192b010c3ef4843a657, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/c7e68081a0b6420f9c44f824ad37f7b3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/81f6c39659d44b21b852dabd3fe61367, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/3a3659c1af094445ab396368c39072c3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/b1986deddacd46e7ad80e933ae95fe7b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/97740e7a533d48d48f9545510bbd40fc, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/735ec1b9a2434ebe99a97dbd8d1bf4c9, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/22b32f1fd2c1443d89bc83420fb0e6c0, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/1327fe631bc34b4ca1d701b24681e54c, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/c75eb17c981c4c6ba90457539f1efda5, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/dccbfd9f98154d7a8d085fdf03a6394b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/ff04b1f700c94b1d890a7218ff432c42, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/91a6cf96ae604429b467b1a7671095e1, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/cf4f1d9268994c798c00f5b0ac111781, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/99b40c11c87d41df8976d7d0a2829300, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/64dc95e6cc9f460583ecda9b591ee16b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/81d7dc7d1b61499492a7d4ec954be8f4, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/cd57bf153f0a4fdd9ebd1d1f96d1d5c9, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/c50db14ca2e54faaa498b2b0a78ce8d5, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/4034044f4a5f429086ad36e32e6a270e, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/0189d0c32eba464496c161242815e438] to archive 2024-11-21T03:40:54,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:40:54,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/43dffd27eda04baf8b20f32c377438fc to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/43dffd27eda04baf8b20f32c377438fc 2024-11-21T03:40:54,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/a19061041a9a46a9abec7d873814b540 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/a19061041a9a46a9abec7d873814b540 2024-11-21T03:40:54,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/a0fd47f2ad384a7d8a2061aac5e9c3b7 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/a0fd47f2ad384a7d8a2061aac5e9c3b7 2024-11-21T03:40:54,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/83dfbaa3d10c447ca9400d38aedaae7b to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/83dfbaa3d10c447ca9400d38aedaae7b 2024-11-21T03:40:54,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/a8f078debfe74eeb9549f8ac4f882428 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/a8f078debfe74eeb9549f8ac4f882428 2024-11-21T03:40:54,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/9edddfaeee374d2689ca2596fb74495f to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/9edddfaeee374d2689ca2596fb74495f 2024-11-21T03:40:54,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/a0fb333243a24192b010c3ef4843a657 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/a0fb333243a24192b010c3ef4843a657 2024-11-21T03:40:54,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/c7e68081a0b6420f9c44f824ad37f7b3 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/c7e68081a0b6420f9c44f824ad37f7b3 2024-11-21T03:40:54,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/81f6c39659d44b21b852dabd3fe61367 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/81f6c39659d44b21b852dabd3fe61367 2024-11-21T03:40:54,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/3a3659c1af094445ab396368c39072c3 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/3a3659c1af094445ab396368c39072c3 2024-11-21T03:40:54,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/b1986deddacd46e7ad80e933ae95fe7b to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/b1986deddacd46e7ad80e933ae95fe7b 2024-11-21T03:40:54,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/97740e7a533d48d48f9545510bbd40fc to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/97740e7a533d48d48f9545510bbd40fc 2024-11-21T03:40:54,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/735ec1b9a2434ebe99a97dbd8d1bf4c9 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/735ec1b9a2434ebe99a97dbd8d1bf4c9 2024-11-21T03:40:54,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/22b32f1fd2c1443d89bc83420fb0e6c0 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/22b32f1fd2c1443d89bc83420fb0e6c0 2024-11-21T03:40:54,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/1327fe631bc34b4ca1d701b24681e54c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/1327fe631bc34b4ca1d701b24681e54c 2024-11-21T03:40:54,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/c75eb17c981c4c6ba90457539f1efda5 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/c75eb17c981c4c6ba90457539f1efda5 2024-11-21T03:40:54,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/dccbfd9f98154d7a8d085fdf03a6394b to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/dccbfd9f98154d7a8d085fdf03a6394b 2024-11-21T03:40:54,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/ff04b1f700c94b1d890a7218ff432c42 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/ff04b1f700c94b1d890a7218ff432c42 2024-11-21T03:40:54,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/91a6cf96ae604429b467b1a7671095e1 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/91a6cf96ae604429b467b1a7671095e1 2024-11-21T03:40:54,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/cf4f1d9268994c798c00f5b0ac111781 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/cf4f1d9268994c798c00f5b0ac111781 2024-11-21T03:40:54,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/99b40c11c87d41df8976d7d0a2829300 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/99b40c11c87d41df8976d7d0a2829300 2024-11-21T03:40:54,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/64dc95e6cc9f460583ecda9b591ee16b to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/64dc95e6cc9f460583ecda9b591ee16b 2024-11-21T03:40:54,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/81d7dc7d1b61499492a7d4ec954be8f4 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/81d7dc7d1b61499492a7d4ec954be8f4 2024-11-21T03:40:54,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/cd57bf153f0a4fdd9ebd1d1f96d1d5c9 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/cd57bf153f0a4fdd9ebd1d1f96d1d5c9 2024-11-21T03:40:54,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/c50db14ca2e54faaa498b2b0a78ce8d5 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/c50db14ca2e54faaa498b2b0a78ce8d5 2024-11-21T03:40:54,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/4034044f4a5f429086ad36e32e6a270e to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/4034044f4a5f429086ad36e32e6a270e 2024-11-21T03:40:54,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/0189d0c32eba464496c161242815e438 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/0189d0c32eba464496c161242815e438 2024-11-21T03:40:54,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/3d4616d27e974165913fa0a397433395, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/c8c2a8d1760b4e86866c9a0207e03768, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/0878c8009a594825b1649e61478b1d25, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/1fb3e881904f44288da74d02cb746fb9, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/4438bd6e956f4eb090f23447fd267968, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/6d5d3a99e71e46c3b2e3eeeb5401149f, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/f7b533b213884b1f84e3c794495d162a, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/17ec14b7211e40eca3f21cd24ac33c2b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/ee8e3b9f7f35461e9dcd9320c4dce2fe, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/f40e21eb0340418eb12c1c2fd7e20ae3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/02c709253f014dc1b8acdc9aebf3ec2f, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/282b29e526904da4b6e70134a2f5f2e0, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/d229c97300cc4e6bb56b1c24fed9dd1b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/524e4741d93c4b45a56c38669cc2a7d6, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/f56c824f7e7c45348858cc22ac4bb769, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/c1ecb486c6904b639f6cedc510849ab5, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/a75728a5b94444129201d3fd95d21bda, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/b59e09dd28604825803e990aed41ad33, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/8150c3401fe643fb9232345bad0025c6, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/73e3beed7b904a0bade2fc4e4d69a511, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/222510b44acc4f9ebea83207bedba625, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/a0b7015297df4f678197324511758a25, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/fa64d1e1a4f34cc9b7a67fc17aaab849, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/546f41f84cfc4c97af73a4553a478854, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/4cedab1c18c5431d8835ea6d430019d9, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/b2e518d05c00462496e9405071ca0826, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/3cdcd27719aa46fe9ff4340814031c59] to archive 2024-11-21T03:40:54,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:40:54,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/3d4616d27e974165913fa0a397433395 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/3d4616d27e974165913fa0a397433395 2024-11-21T03:40:54,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/c8c2a8d1760b4e86866c9a0207e03768 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/c8c2a8d1760b4e86866c9a0207e03768 2024-11-21T03:40:54,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/0878c8009a594825b1649e61478b1d25 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/0878c8009a594825b1649e61478b1d25 2024-11-21T03:40:54,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/1fb3e881904f44288da74d02cb746fb9 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/1fb3e881904f44288da74d02cb746fb9 2024-11-21T03:40:54,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/4438bd6e956f4eb090f23447fd267968 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/4438bd6e956f4eb090f23447fd267968 2024-11-21T03:40:54,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/6d5d3a99e71e46c3b2e3eeeb5401149f to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/6d5d3a99e71e46c3b2e3eeeb5401149f 2024-11-21T03:40:54,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/f7b533b213884b1f84e3c794495d162a to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/f7b533b213884b1f84e3c794495d162a 2024-11-21T03:40:54,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/17ec14b7211e40eca3f21cd24ac33c2b to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/17ec14b7211e40eca3f21cd24ac33c2b 2024-11-21T03:40:54,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/ee8e3b9f7f35461e9dcd9320c4dce2fe to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/ee8e3b9f7f35461e9dcd9320c4dce2fe 2024-11-21T03:40:54,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/f40e21eb0340418eb12c1c2fd7e20ae3 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/f40e21eb0340418eb12c1c2fd7e20ae3 2024-11-21T03:40:54,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/02c709253f014dc1b8acdc9aebf3ec2f to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/02c709253f014dc1b8acdc9aebf3ec2f 2024-11-21T03:40:54,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/282b29e526904da4b6e70134a2f5f2e0 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/282b29e526904da4b6e70134a2f5f2e0 2024-11-21T03:40:54,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/d229c97300cc4e6bb56b1c24fed9dd1b to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/d229c97300cc4e6bb56b1c24fed9dd1b 2024-11-21T03:40:54,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/524e4741d93c4b45a56c38669cc2a7d6 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/524e4741d93c4b45a56c38669cc2a7d6 2024-11-21T03:40:54,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/f56c824f7e7c45348858cc22ac4bb769 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/f56c824f7e7c45348858cc22ac4bb769 2024-11-21T03:40:54,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/c1ecb486c6904b639f6cedc510849ab5 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/c1ecb486c6904b639f6cedc510849ab5 2024-11-21T03:40:54,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/a75728a5b94444129201d3fd95d21bda to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/a75728a5b94444129201d3fd95d21bda 2024-11-21T03:40:54,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/b59e09dd28604825803e990aed41ad33 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/b59e09dd28604825803e990aed41ad33 2024-11-21T03:40:54,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/8150c3401fe643fb9232345bad0025c6 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/8150c3401fe643fb9232345bad0025c6 2024-11-21T03:40:54,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/73e3beed7b904a0bade2fc4e4d69a511 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/73e3beed7b904a0bade2fc4e4d69a511 2024-11-21T03:40:54,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/222510b44acc4f9ebea83207bedba625 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/222510b44acc4f9ebea83207bedba625 2024-11-21T03:40:54,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/a0b7015297df4f678197324511758a25 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/a0b7015297df4f678197324511758a25 2024-11-21T03:40:54,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/fa64d1e1a4f34cc9b7a67fc17aaab849 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/fa64d1e1a4f34cc9b7a67fc17aaab849 2024-11-21T03:40:54,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/546f41f84cfc4c97af73a4553a478854 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/546f41f84cfc4c97af73a4553a478854 2024-11-21T03:40:54,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/4cedab1c18c5431d8835ea6d430019d9 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/4cedab1c18c5431d8835ea6d430019d9 2024-11-21T03:40:54,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/b2e518d05c00462496e9405071ca0826 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/b2e518d05c00462496e9405071ca0826 2024-11-21T03:40:54,904 DEBUG [StoreCloser-TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/3cdcd27719aa46fe9ff4340814031c59 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/3cdcd27719aa46fe9ff4340814031c59 2024-11-21T03:40:54,908 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/recovered.edits/395.seqid, newMaxSeqId=395, maxSeqId=4 2024-11-21T03:40:54,909 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. 2024-11-21T03:40:54,909 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1676): Region close journal for 9a822f4697b07ce815f004ac9ea14bb2: Waiting for close lock at 1732160453404Running coprocessor pre-close hooks at 1732160453404Disabling compacts and flushes for region at 1732160453404Disabling writes for close at 1732160453540 (+136 ms)Obtaining lock to block concurrent updates at 1732160453540Preparing flush snapshotting stores in 9a822f4697b07ce815f004ac9ea14bb2 at 1732160453540Finished memstore snapshotting TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1732160453540Flushing stores of TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2. at 1732160453541 (+1 ms)Flushing 9a822f4697b07ce815f004ac9ea14bb2/A: creating writer at 1732160453541Flushing mob file 9a822f4697b07ce815f004ac9ea14bb2/A: appending metadata at 1732160453551 (+10 ms)Flushing mob file 9a822f4697b07ce815f004ac9ea14bb2/A: closing flushed file at 1732160453551Flushing 9a822f4697b07ce815f004ac9ea14bb2/A: appending metadata at 1732160453566 (+15 ms)Flushing 9a822f4697b07ce815f004ac9ea14bb2/A: closing flushed file at 1732160453566Flushing 9a822f4697b07ce815f004ac9ea14bb2/B: creating writer at 1732160453976 (+410 ms)Flushing 9a822f4697b07ce815f004ac9ea14bb2/B: appending metadata at 1732160453979 (+3 ms)Flushing 9a822f4697b07ce815f004ac9ea14bb2/B: closing flushed file at 1732160453979Flushing 9a822f4697b07ce815f004ac9ea14bb2/C: creating writer at 1732160454387 (+408 ms)Flushing 9a822f4697b07ce815f004ac9ea14bb2/C: appending metadata at 1732160454390 (+3 ms)Flushing 9a822f4697b07ce815f004ac9ea14bb2/C: closing flushed file at 1732160454390Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48871f86: reopening flushed file at 1732160454799 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e534867: reopening flushed file at 1732160454804 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1914ce94: reopening flushed file at 1732160454810 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 9a822f4697b07ce815f004ac9ea14bb2 in 1275ms, sequenceid=392, compaction requested=false at 1732160454815 (+5 ms)Writing region close event to WAL at 1732160454905 (+90 ms)Running coprocessor post-close hooks at 1732160454908 (+3 ms)Closed at 1732160454909 (+1 ms) 2024-11-21T03:40:54,911 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(157): Closed 9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:54,911 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=9a822f4697b07ce815f004ac9ea14bb2, regionState=CLOSED 2024-11-21T03:40:54,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 because future has completed 2024-11-21T03:40:54,915 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-21T03:40:54,915 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; CloseRegionProcedure 9a822f4697b07ce815f004ac9ea14bb2, server=fdc0a393186c,34631,1732160396889 in 1.6630 sec 2024-11-21T03:40:54,917 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-21T03:40:54,917 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=9a822f4697b07ce815f004ac9ea14bb2, UNASSIGN in 1.6790 sec 2024-11-21T03:40:54,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-21T03:40:54,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6830 sec 2024-11-21T03:40:54,920 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732160454920"}]},"ts":"1732160454920"} 2024-11-21T03:40:54,922 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T03:40:54,922 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T03:40:54,923 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6950 sec 2024-11-21T03:40:55,192 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T03:40:55,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-21T03:40:55,377 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:55,378 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T03:40:55,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:40:55,379 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:40:55,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T03:40:55,380 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=66, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:40:55,384 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:55,387 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A, FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B, FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C, FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/recovered.edits] 2024-11-21T03:40:55,389 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/5c846a2355d2454eba1d5e765ebc99b2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/5c846a2355d2454eba1d5e765ebc99b2 2024-11-21T03:40:55,390 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/88947262ac5d46c5824be48eb9ddbc7d to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/A/88947262ac5d46c5824be48eb9ddbc7d 2024-11-21T03:40:55,392 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/5921565f533044d5bdc6a3a209726bce to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/5921565f533044d5bdc6a3a209726bce 2024-11-21T03:40:55,393 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/7dee3aee203d47598ba5a866597aae46 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/B/7dee3aee203d47598ba5a866597aae46 2024-11-21T03:40:55,396 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/66e6a48a7fa343bf9e752d049799e9b6 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/66e6a48a7fa343bf9e752d049799e9b6 2024-11-21T03:40:55,397 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/869e9ca2c91043e2b48649de82edbf39 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/C/869e9ca2c91043e2b48649de82edbf39 2024-11-21T03:40:55,399 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/recovered.edits/395.seqid to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2/recovered.edits/395.seqid 2024-11-21T03:40:55,400 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:55,400 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T03:40:55,400 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T03:40:55,401 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-21T03:40:55,404 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112113b9597769fa45a9a60cbceef012094e_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112113b9597769fa45a9a60cbceef012094e_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:55,405 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212a84204128654629802c988773ee4726_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212a84204128654629802c988773ee4726_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:55,406 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121406b4b62af2c4f479a0f69218757dc1b_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121406b4b62af2c4f479a0f69218757dc1b_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:55,407 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216704ef3f956a4c018005929160401d77_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216704ef3f956a4c018005929160401d77_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:55,408 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121695bfaa2ccfb43df924712df1f35cdfb_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121695bfaa2ccfb43df924712df1f35cdfb_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:55,409 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216980f433a0724ce18868d4b3e3dbad1b_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216980f433a0724ce18868d4b3e3dbad1b_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:55,410 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216a2f3856e4ce458bb113f067854fc218_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216a2f3856e4ce458bb113f067854fc218_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:55,412 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216dd2c2faad3a476fb119a408d4b159a9_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216dd2c2faad3a476fb119a408d4b159a9_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:55,412 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121719d543f28d144a58efd4f4a7711f2f8_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121719d543f28d144a58efd4f4a7711f2f8_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:55,413 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112177996bcea5c94d4f9a091659c8e6150a_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112177996bcea5c94d4f9a091659c8e6150a_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:55,414 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121816cd966b61640af859b4fb014b9c566_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121816cd966b61640af859b4fb014b9c566_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:55,416 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121986cc79522dc44caaa7c8c7b1a5d7f6c_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121986cc79522dc44caaa7c8c7b1a5d7f6c_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:55,417 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219e9779473c0f4f31ba72a14c63b99d42_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219e9779473c0f4f31ba72a14c63b99d42_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:55,418 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a229e77cad22470cb74c44c0ea16e430_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a229e77cad22470cb74c44c0ea16e430_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:55,419 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a39ce109b780465c8146760557759e94_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a39ce109b780465c8146760557759e94_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:55,419 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a79b0620174b44399359ffd2917d0bb0_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a79b0620174b44399359ffd2917d0bb0_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:55,420 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a7d2990f32a24106a0df459801c1847c_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a7d2990f32a24106a0df459801c1847c_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:55,421 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a99252b3bab74c1cb9fa8666bb0467aa_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a99252b3bab74c1cb9fa8666bb0467aa_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:55,423 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bf21173b9f304886bb372b24de0c4b61_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bf21173b9f304886bb372b24de0c4b61_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:55,424 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c50091405db045b7ad48bc9d51b30a7f_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c50091405db045b7ad48bc9d51b30a7f_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:55,424 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ffb3e052f0f942eabaedfa42ad80fd61_9a822f4697b07ce815f004ac9ea14bb2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121ffb3e052f0f942eabaedfa42ad80fd61_9a822f4697b07ce815f004ac9ea14bb2 2024-11-21T03:40:55,425 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T03:40:55,427 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=66, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:40:55,429 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T03:40:55,430 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T03:40:55,431 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=66, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:40:55,431 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T03:40:55,432 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732160455431"}]},"ts":"9223372036854775807"} 2024-11-21T03:40:55,434 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T03:40:55,434 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 9a822f4697b07ce815f004ac9ea14bb2, NAME => 'TestAcidGuarantees,,1732160428438.9a822f4697b07ce815f004ac9ea14bb2.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T03:40:55,434 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T03:40:55,434 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732160455434"}]},"ts":"9223372036854775807"} 2024-11-21T03:40:55,436 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T03:40:55,437 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=66, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:40:55,438 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 59 msec 2024-11-21T03:40:55,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T03:40:55,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T03:40:55,487 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:55,499 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=242 (was 239) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/cluster_9cc19675-7463-4642-6f00-3e0c0079ede6/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/bd0515af-6624-bb57-cd8b-83f55f5a9537/cluster_9cc19675-7463-4642-6f00-3e0c0079ede6/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-677494979_22 at /127.0.0.1:53940 [Waiting for operation #266] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_263764961_22 at /127.0.0.1:45966 [Waiting for operation #638] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-677494979_22 at /127.0.0.1:53956 [Waiting for operation #249] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_263764961_22 at /127.0.0.1:58994 [Waiting for operation #660] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=613 (was 521) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6373 (was 7202) 2024-11-21T03:40:55,510 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=613, ProcessCount=11, AvailableMemoryMB=6373 2024-11-21T03:40:55,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T03:40:55,512 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T03:40:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T03:40:55,514 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T03:40:55,515 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:55,515 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 67 2024-11-21T03:40:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-21T03:40:55,516 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T03:40:55,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742073_1249 (size=959) 2024-11-21T03:40:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-21T03:40:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-21T03:40:55,924 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0 2024-11-21T03:40:55,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742074_1250 (size=53) 2024-11-21T03:40:56,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-21T03:40:56,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:40:56,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 17b90c50c9bd5dda59fffca9821d39a4, disabling compactions & flushes 2024-11-21T03:40:56,333 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:40:56,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:40:56,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. after waiting 0 ms 2024-11-21T03:40:56,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:40:56,333 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:40:56,333 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 17b90c50c9bd5dda59fffca9821d39a4: Waiting for close lock at 1732160456333Disabling compacts and flushes for region at 1732160456333Disabling writes for close at 1732160456333Writing region close event to WAL at 1732160456333Closed at 1732160456333 2024-11-21T03:40:56,334 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T03:40:56,335 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732160456334"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732160456334"}]},"ts":"1732160456334"} 2024-11-21T03:40:56,337 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T03:40:56,338 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T03:40:56,338 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732160456338"}]},"ts":"1732160456338"} 2024-11-21T03:40:56,339 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T03:40:56,340 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17b90c50c9bd5dda59fffca9821d39a4, ASSIGN}] 2024-11-21T03:40:56,341 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17b90c50c9bd5dda59fffca9821d39a4, ASSIGN 2024-11-21T03:40:56,342 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=17b90c50c9bd5dda59fffca9821d39a4, ASSIGN; state=OFFLINE, location=fdc0a393186c,34631,1732160396889; forceNewPlan=false, retain=false 2024-11-21T03:40:56,467 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-21T03:40:56,493 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=17b90c50c9bd5dda59fffca9821d39a4, regionState=OPENING, regionLocation=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:56,494 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=17b90c50c9bd5dda59fffca9821d39a4, ASSIGN because future has completed 2024-11-21T03:40:56,494 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889}] 2024-11-21T03:40:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-21T03:40:56,689 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:40:56,689 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7752): Opening region: {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:40:56,689 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:40:56,690 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:40:56,690 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7794): checking encryption for 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:40:56,690 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7797): checking classloading for 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:40:56,691 INFO [StoreOpener-17b90c50c9bd5dda59fffca9821d39a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:40:56,692 INFO [StoreOpener-17b90c50c9bd5dda59fffca9821d39a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17b90c50c9bd5dda59fffca9821d39a4 columnFamilyName A 2024-11-21T03:40:56,692 DEBUG [StoreOpener-17b90c50c9bd5dda59fffca9821d39a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:56,693 INFO [StoreOpener-17b90c50c9bd5dda59fffca9821d39a4-1 {}] regionserver.HStore(327): Store=17b90c50c9bd5dda59fffca9821d39a4/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:40:56,693 INFO [StoreOpener-17b90c50c9bd5dda59fffca9821d39a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:40:56,694 INFO [StoreOpener-17b90c50c9bd5dda59fffca9821d39a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17b90c50c9bd5dda59fffca9821d39a4 columnFamilyName B 2024-11-21T03:40:56,694 DEBUG [StoreOpener-17b90c50c9bd5dda59fffca9821d39a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:56,694 INFO [StoreOpener-17b90c50c9bd5dda59fffca9821d39a4-1 {}] regionserver.HStore(327): Store=17b90c50c9bd5dda59fffca9821d39a4/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:40:56,694 INFO [StoreOpener-17b90c50c9bd5dda59fffca9821d39a4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:40:56,695 INFO [StoreOpener-17b90c50c9bd5dda59fffca9821d39a4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17b90c50c9bd5dda59fffca9821d39a4 columnFamilyName C 2024-11-21T03:40:56,695 DEBUG [StoreOpener-17b90c50c9bd5dda59fffca9821d39a4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:40:56,695 INFO [StoreOpener-17b90c50c9bd5dda59fffca9821d39a4-1 {}] regionserver.HStore(327): Store=17b90c50c9bd5dda59fffca9821d39a4/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:40:56,696 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1038): replaying wal for 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:40:56,696 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:40:56,697 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:40:56,697 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1048): stopping wal replay for 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:40:56,697 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1060): Cleaning up temporary data for 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:40:56,698 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T03:40:56,699 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1093): writing seq id for 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:40:56,701 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T03:40:56,701 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1114): Opened 17b90c50c9bd5dda59fffca9821d39a4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72592360, jitterRate=0.08171045780181885}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T03:40:56,701 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:40:56,702 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1006): Region open journal for 17b90c50c9bd5dda59fffca9821d39a4: Running coprocessor pre-open hook at 1732160456690Writing region info on filesystem at 1732160456690Initializing all the Stores at 1732160456691 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732160456691Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732160456691Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732160456691Cleaning up temporary data from old regions at 1732160456698 (+7 ms)Running coprocessor post-open hooks at 1732160456701 (+3 ms)Region opened successfully at 1732160456702 (+1 ms) 2024-11-21T03:40:56,703 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., pid=69, masterSystemTime=1732160456645 2024-11-21T03:40:56,704 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:40:56,704 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:40:56,705 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=17b90c50c9bd5dda59fffca9821d39a4, regionState=OPEN, openSeqNum=2, regionLocation=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:56,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 because future has completed 2024-11-21T03:40:56,708 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-21T03:40:56,708 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 in 213 msec 2024-11-21T03:40:56,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=68, resume processing ppid=67 2024-11-21T03:40:56,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=67, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17b90c50c9bd5dda59fffca9821d39a4, ASSIGN in 368 msec 2024-11-21T03:40:56,710 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T03:40:56,710 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732160456710"}]},"ts":"1732160456710"} 2024-11-21T03:40:56,711 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T03:40:56,712 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T03:40:56,713 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2000 sec 2024-11-21T03:40:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-21T03:40:57,668 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:57,671 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d8da64d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,671 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:57,671 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:57,674 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:57,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ae69d98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:57,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:57,676 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44734, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:57,677 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d0b352, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,677 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:57,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:57,678 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:57,679 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:57,680 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T03:40:57,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is fdc0a393186c,44285,1732160396080 2024-11-21T03:40:57,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@17824b2f 2024-11-21T03:40:57,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T03:40:57,682 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T03:40:57,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,683 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:57,683 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:57,684 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:57,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54a0725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:57,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:57,686 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44766, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@132f9c2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,687 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:57,687 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:57,687 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:57,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:57,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:57,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60475b56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:57,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:57,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:57,689 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:57,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@375b5524, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,690 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,690 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:57,690 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:57,690 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:57,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:57,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:57,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@112ef56b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:57,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:57,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:57,692 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:57,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d7a0b26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,693 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:57,693 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:57,693 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:57,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c3fabb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:57,695 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44838, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24cac327, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,696 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19903550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,696 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:57,696 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:57,696 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:57,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:57,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:57,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b6e776b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:57,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:57,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:57,698 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:57,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@161bba2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,699 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,699 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:57,699 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:57,699 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9c19cbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:57,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:57,701 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44888, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:57,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@774b2540, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,702 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:57,702 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:57,702 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:57,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7400b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:57,703 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44902, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:57,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f970279, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,704 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:57,704 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:57,705 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:57,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:57,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:57,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1929c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:57,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:57,705 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:57,706 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:57,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d7700c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,707 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:57,707 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:57,708 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:57,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:57,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:57,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a0b6e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:57,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:57,708 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:57,709 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:57,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@297435b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,710 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40916ac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,710 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:40:57,710 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:40:57,710 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:40:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:40:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:40:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50baf510, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:40:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:40:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:40:57,712 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44934, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:40:57,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66e21901, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:40:57,712 DEBUG [Thread-1174 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:57,713 DEBUG [Thread-1159 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:57,713 DEBUG [Thread-1168 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:57,713 DEBUG [Thread-1163 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:57,713 DEBUG [Thread-1172 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:57,713 DEBUG [Thread-1165 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:57,713 DEBUG [Thread-1176 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:57,713 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:40:57,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:57,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:57,714 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:57,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:57,714 DEBUG [Thread-1157 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:57,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:57,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:57,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:57,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:57,715 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:57,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:57,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-21T03:40:57,715 DEBUG [Thread-1170 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:57,716 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50888, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:57,716 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:57,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T03:40:57,716 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:57,716 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:57,716 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:57,716 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:40:57,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:57,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:57,717 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:57,717 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:40:57,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:40:57,718 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50940, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:57,718 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50932, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:57,718 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:57,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:40:57,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:40:57,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:57,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:57,719 DEBUG [Thread-1161 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:40:57,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:40:57,720 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50960, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:57,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:40:57,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:40:57,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:40:57,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:40:57,722 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:40:57,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:40:57,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:40:57,724 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:40:57,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:40:57,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:40:57,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:40:57,729 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:57,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17b90c50c9bd5dda59fffca9821d39a4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:40:57,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/b720ee9956ab4d89a0b747c04f17cf78 is 50, key is test_row_0/A:col10/1732160457729/Put/seqid=0 2024-11-21T03:40:57,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:57,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50918 deadline: 1732160517773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:57,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:57,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:57,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:57,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:57,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50890 deadline: 1732160517776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:57,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:57,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:57,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:57,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:57,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732160517778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:57,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:57,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:57,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:57,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:57,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160517782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:57,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:57,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:57,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:57,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:57,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160517783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:57,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:57,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:57,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:57,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742075_1251 (size=14341) 2024-11-21T03:40:57,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T03:40:57,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T03:40:57,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:40:57,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:40:57,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:40:57,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:57,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:57,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:57,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:57,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50918 deadline: 1732160517886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:57,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:57,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:57,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:57,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50890 deadline: 1732160517896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:57,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732160517896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:57,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160517897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:57,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:57,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:57,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:57,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:57,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160517907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:57,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:57,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:57,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:58,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T03:40:58,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:40:58,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:40:58,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:40:58,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:58,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:58,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:58,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T03:40:58,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:58,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50918 deadline: 1732160518097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:58,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:58,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:58,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732160518107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:58,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50890 deadline: 1732160518107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:58,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160518107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:58,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:58,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:58,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160518116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:58,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:58,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T03:40:58,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:40:58,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:40:58,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:40:58,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:58,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:58,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:58,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/b720ee9956ab4d89a0b747c04f17cf78 2024-11-21T03:40:58,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/268b07ba64894973ac4870e0d1f7f02a is 50, key is test_row_0/B:col10/1732160457729/Put/seqid=0 2024-11-21T03:40:58,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742076_1252 (size=12001) 2024-11-21T03:40:58,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T03:40:58,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:40:58,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:40:58,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:40:58,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:58,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T03:40:58,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50918 deadline: 1732160518406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:58,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732160518417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:58,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50890 deadline: 1732160518417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160518417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:58,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160518427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:58,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:58,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T03:40:58,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:40:58,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:40:58,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:40:58,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:58,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:58,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T03:40:58,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:40:58,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:40:58,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:40:58,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:58,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:58,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:40:58,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/268b07ba64894973ac4870e0d1f7f02a 2024-11-21T03:40:58,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/2443b8629d6f40faa074c022a2ccfe2b is 50, key is test_row_0/C:col10/1732160457729/Put/seqid=0 2024-11-21T03:40:58,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742077_1253 (size=12001) 2024-11-21T03:40:58,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/2443b8629d6f40faa074c022a2ccfe2b 2024-11-21T03:40:58,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/b720ee9956ab4d89a0b747c04f17cf78 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/b720ee9956ab4d89a0b747c04f17cf78 2024-11-21T03:40:58,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/b720ee9956ab4d89a0b747c04f17cf78, entries=200, sequenceid=13, filesize=14.0 K 2024-11-21T03:40:58,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/268b07ba64894973ac4870e0d1f7f02a as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/268b07ba64894973ac4870e0d1f7f02a 2024-11-21T03:40:58,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/268b07ba64894973ac4870e0d1f7f02a, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T03:40:58,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/2443b8629d6f40faa074c022a2ccfe2b as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/2443b8629d6f40faa074c022a2ccfe2b 2024-11-21T03:40:58,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/2443b8629d6f40faa074c022a2ccfe2b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T03:40:58,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 17b90c50c9bd5dda59fffca9821d39a4 in 997ms, sequenceid=13, compaction requested=false 2024-11-21T03:40:58,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:40:58,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T03:40:58,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:40:58,789 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 17b90c50c9bd5dda59fffca9821d39a4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T03:40:58,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/438c360745f945e9806246b82bcb0db4 is 50, key is test_row_0/A:col10/1732160457779/Put/seqid=0 2024-11-21T03:40:58,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742078_1254 (size=12001) 2024-11-21T03:40:58,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T03:40:58,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:40:58,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:40:58,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:58,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50890 deadline: 1732160518931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:58,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160518931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:58,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:58,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:58,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50918 deadline: 1732160518931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:58,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:58,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:58,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732160518934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:58,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:58,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:58,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160518936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:58,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:58,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:59,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:59,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50890 deadline: 1732160519037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:59,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:59,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:59,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160519037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:59,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:59,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:59,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:59,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:59,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50918 deadline: 1732160519037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:59,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:59,039 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:59,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:59,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:59,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:59,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:59,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732160519047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:59,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:59,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:59,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:59,224 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/438c360745f945e9806246b82bcb0db4 2024-11-21T03:40:59,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/21057cba9305408a9965f8ecb909145b is 50, key is test_row_0/B:col10/1732160457779/Put/seqid=0 2024-11-21T03:40:59,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742079_1255 (size=12001) 2024-11-21T03:40:59,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50890 deadline: 1732160519247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:59,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:59,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:59,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:59,248 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160519247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:59,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:59,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:59,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:59,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50918 deadline: 1732160519247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:59,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:59,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:59,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:59,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:59,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732160519256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:59,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:59,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:59,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50890 deadline: 1732160519556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160519556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:59,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:59,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:59,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:59,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:59,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50918 deadline: 1732160519557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:59,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:59,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:59,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:59,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:59,559 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:59,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:40:59,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732160519567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:40:59,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:59,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:40:59,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:40:59,640 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/21057cba9305408a9965f8ecb909145b 2024-11-21T03:40:59,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/7ed7e241f59740be94f449da7f633665 is 50, key is test_row_0/C:col10/1732160457779/Put/seqid=0 2024-11-21T03:40:59,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742080_1256 (size=12001) 2024-11-21T03:40:59,651 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/7ed7e241f59740be94f449da7f633665 2024-11-21T03:40:59,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/438c360745f945e9806246b82bcb0db4 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/438c360745f945e9806246b82bcb0db4 2024-11-21T03:40:59,660 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/438c360745f945e9806246b82bcb0db4, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T03:40:59,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/21057cba9305408a9965f8ecb909145b as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/21057cba9305408a9965f8ecb909145b 2024-11-21T03:40:59,665 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/21057cba9305408a9965f8ecb909145b, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T03:40:59,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/7ed7e241f59740be94f449da7f633665 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/7ed7e241f59740be94f449da7f633665 2024-11-21T03:40:59,671 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/7ed7e241f59740be94f449da7f633665, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T03:40:59,671 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 17b90c50c9bd5dda59fffca9821d39a4 in 882ms, sequenceid=37, compaction requested=false 2024-11-21T03:40:59,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:40:59,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:40:59,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-21T03:40:59,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-21T03:40:59,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-21T03:40:59,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9560 sec 2024-11-21T03:40:59,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.9620 sec 2024-11-21T03:40:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T03:40:59,867 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:40:59,868 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:40:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-21T03:40:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T03:40:59,870 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:40:59,870 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:40:59,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:40:59,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:40:59,950 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:40:59,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17b90c50c9bd5dda59fffca9821d39a4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T03:40:59,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/076171dc30354e41b5c95bdae099f047 is 50, key is test_row_0/A:col10/1732160458929/Put/seqid=0 2024-11-21T03:40:59,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T03:40:59,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742081_1257 (size=14341) 2024-11-21T03:40:59,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/076171dc30354e41b5c95bdae099f047 2024-11-21T03:41:00,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/78c649a3cdbc4681889d4f4e82aaca9d is 50, key is test_row_0/B:col10/1732160458929/Put/seqid=0 2024-11-21T03:41:00,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T03:41:00,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:00,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:00,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:00,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:00,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:00,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:00,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742082_1258 (size=12001) 2024-11-21T03:41:00,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:00,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50918 deadline: 1732160520067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:00,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50890 deadline: 1732160520067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:00,073 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:00,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:00,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160520071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:00,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:00,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:00,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:00,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:00,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160520073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:00,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:00,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:00,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:00,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:00,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732160520076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:00,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:00,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:00,081 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:00,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T03:41:00,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:00,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:00,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:00,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:00,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T03:41:00,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:00,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160520187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:00,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:00,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:00,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160520187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:00,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:00,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:00,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:00,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:00,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:00,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T03:41:00,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:00,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:00,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:00,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:00,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:00,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:00,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160520397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:00,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:00,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:00,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160520397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:00,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:00,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:00,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:00,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:00,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:00,442 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T03:41:00,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/78c649a3cdbc4681889d4f4e82aaca9d 2024-11-21T03:41:00,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/20c792e5886548ee94c342071a58cb22 is 50, key is test_row_0/C:col10/1732160458929/Put/seqid=0 2024-11-21T03:41:00,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742083_1259 (size=12001) 2024-11-21T03:41:00,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T03:41:00,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:00,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:00,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:00,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:00,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:00,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/20c792e5886548ee94c342071a58cb22 2024-11-21T03:41:00,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/076171dc30354e41b5c95bdae099f047 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/076171dc30354e41b5c95bdae099f047 2024-11-21T03:41:00,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T03:41:00,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/076171dc30354e41b5c95bdae099f047, entries=200, sequenceid=50, filesize=14.0 K 2024-11-21T03:41:00,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/78c649a3cdbc4681889d4f4e82aaca9d as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/78c649a3cdbc4681889d4f4e82aaca9d 2024-11-21T03:41:00,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/78c649a3cdbc4681889d4f4e82aaca9d, entries=150, sequenceid=50, filesize=11.7 K 2024-11-21T03:41:00,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/20c792e5886548ee94c342071a58cb22 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/20c792e5886548ee94c342071a58cb22 2024-11-21T03:41:00,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/20c792e5886548ee94c342071a58cb22, entries=150, sequenceid=50, filesize=11.7 K 2024-11-21T03:41:00,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 17b90c50c9bd5dda59fffca9821d39a4 in 564ms, sequenceid=50, compaction requested=true 2024-11-21T03:41:00,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:00,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:41:00,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:00,515 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:00,515 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:00,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:41:00,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:00,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:41:00,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:00,516 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:00,516 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/B is initiating minor compaction (all files) 2024-11-21T03:41:00,516 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/B in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:00,516 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/268b07ba64894973ac4870e0d1f7f02a, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/21057cba9305408a9965f8ecb909145b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/78c649a3cdbc4681889d4f4e82aaca9d] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=35.2 K 2024-11-21T03:41:00,516 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:00,516 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/A is initiating minor compaction (all files) 2024-11-21T03:41:00,516 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/A in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:00,517 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/b720ee9956ab4d89a0b747c04f17cf78, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/438c360745f945e9806246b82bcb0db4, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/076171dc30354e41b5c95bdae099f047] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=39.7 K 2024-11-21T03:41:00,517 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting b720ee9956ab4d89a0b747c04f17cf78, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732160457724 2024-11-21T03:41:00,517 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 268b07ba64894973ac4870e0d1f7f02a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732160457724 2024-11-21T03:41:00,518 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 438c360745f945e9806246b82bcb0db4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732160457763 2024-11-21T03:41:00,518 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 21057cba9305408a9965f8ecb909145b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732160457763 2024-11-21T03:41:00,518 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 78c649a3cdbc4681889d4f4e82aaca9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732160458929 2024-11-21T03:41:00,518 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 076171dc30354e41b5c95bdae099f047, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732160458929 2024-11-21T03:41:00,533 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#A#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:00,533 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/d663966aa82a4c2fba608e5f9881ffb1 is 50, key is test_row_0/A:col10/1732160458929/Put/seqid=0 2024-11-21T03:41:00,536 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#B#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:00,536 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/c4245b08c7654752a0d8f182b9524903 is 50, key is test_row_0/B:col10/1732160458929/Put/seqid=0 2024-11-21T03:41:00,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742084_1260 (size=12104) 2024-11-21T03:41:00,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742085_1261 (size=12104) 2024-11-21T03:41:00,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T03:41:00,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:00,639 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 17b90c50c9bd5dda59fffca9821d39a4 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T03:41:00,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/a3165b39efeb47498e8a3965aa0517d6 is 50, key is test_row_0/A:col10/1732160460069/Put/seqid=0 2024-11-21T03:41:00,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742086_1262 (size=12001) 2024-11-21T03:41:00,647 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/a3165b39efeb47498e8a3965aa0517d6 2024-11-21T03:41:00,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/d42af8e559a54d998bec7b6fea0d0bc3 is 50, key is test_row_0/B:col10/1732160460069/Put/seqid=0 2024-11-21T03:41:00,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742087_1263 (size=12001) 2024-11-21T03:41:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:41:00,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:00,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160520725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:00,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160520725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:00,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:00,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:00,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160520837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:00,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:00,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160520837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:00,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:00,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:00,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:00,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:00,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:00,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:00,956 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/d663966aa82a4c2fba608e5f9881ffb1 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/d663966aa82a4c2fba608e5f9881ffb1 2024-11-21T03:41:00,960 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/c4245b08c7654752a0d8f182b9524903 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/c4245b08c7654752a0d8f182b9524903 2024-11-21T03:41:00,961 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/A of 17b90c50c9bd5dda59fffca9821d39a4 into d663966aa82a4c2fba608e5f9881ffb1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:00,961 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:00,961 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/A, priority=13, startTime=1732160460514; duration=0sec 2024-11-21T03:41:00,961 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:00,961 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:A 2024-11-21T03:41:00,961 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:00,962 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:00,962 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/C is initiating minor compaction (all files) 2024-11-21T03:41:00,962 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/C in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:00,962 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/2443b8629d6f40faa074c022a2ccfe2b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/7ed7e241f59740be94f449da7f633665, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/20c792e5886548ee94c342071a58cb22] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=35.2 K 2024-11-21T03:41:00,963 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2443b8629d6f40faa074c022a2ccfe2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732160457724 2024-11-21T03:41:00,963 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ed7e241f59740be94f449da7f633665, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732160457763 2024-11-21T03:41:00,964 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20c792e5886548ee94c342071a58cb22, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732160458929 2024-11-21T03:41:00,965 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/B of 17b90c50c9bd5dda59fffca9821d39a4 into c4245b08c7654752a0d8f182b9524903(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:00,965 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:00,965 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/B, priority=13, startTime=1732160460515; duration=0sec 2024-11-21T03:41:00,965 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:00,965 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:B 2024-11-21T03:41:00,971 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#C#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:00,971 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/5f15c7cc40c14033a8a44617bb9959b9 is 50, key is test_row_0/C:col10/1732160458929/Put/seqid=0 2024-11-21T03:41:00,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742088_1264 (size=12104) 2024-11-21T03:41:01,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T03:41:01,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160521046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:01,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:01,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:01,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160521046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:01,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:01,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:01,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:01,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:01,048 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:01,057 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/d42af8e559a54d998bec7b6fea0d0bc3 2024-11-21T03:41:01,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/ebd2ac6e8661449c885f3b19e3eb1439 is 50, key is test_row_0/C:col10/1732160460069/Put/seqid=0 2024-11-21T03:41:01,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742089_1265 (size=12001) 2024-11-21T03:41:01,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:01,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50890 deadline: 1732160521077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:01,080 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_2' in 17b90c50c9bd5dda59fffca9821d39a4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:41:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:01,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:01,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:01,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50918 deadline: 1732160521087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:01,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:01,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732160521087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:01,090 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_0' in 17b90c50c9bd5dda59fffca9821d39a4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:41:01,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:01,090 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_1' in 17b90c50c9bd5dda59fffca9821d39a4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:41:01,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:01,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:01,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:01,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:01,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:01,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:01,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160521357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:01,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:01,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:01,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:01,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:01,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160521357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:01,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:01,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:01,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:01,382 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/5f15c7cc40c14033a8a44617bb9959b9 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/5f15c7cc40c14033a8a44617bb9959b9 2024-11-21T03:41:01,387 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/C of 17b90c50c9bd5dda59fffca9821d39a4 into 5f15c7cc40c14033a8a44617bb9959b9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:01,387 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:01,387 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/C, priority=13, startTime=1732160460515; duration=0sec 2024-11-21T03:41:01,387 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:01,387 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:C 2024-11-21T03:41:01,468 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/ebd2ac6e8661449c885f3b19e3eb1439 2024-11-21T03:41:01,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/a3165b39efeb47498e8a3965aa0517d6 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/a3165b39efeb47498e8a3965aa0517d6 2024-11-21T03:41:01,477 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/a3165b39efeb47498e8a3965aa0517d6, entries=150, sequenceid=73, filesize=11.7 K 2024-11-21T03:41:01,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/d42af8e559a54d998bec7b6fea0d0bc3 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/d42af8e559a54d998bec7b6fea0d0bc3 2024-11-21T03:41:01,482 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/d42af8e559a54d998bec7b6fea0d0bc3, entries=150, sequenceid=73, filesize=11.7 K 2024-11-21T03:41:01,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/ebd2ac6e8661449c885f3b19e3eb1439 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/ebd2ac6e8661449c885f3b19e3eb1439 2024-11-21T03:41:01,487 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/ebd2ac6e8661449c885f3b19e3eb1439, entries=150, sequenceid=73, filesize=11.7 K 2024-11-21T03:41:01,488 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 17b90c50c9bd5dda59fffca9821d39a4 in 849ms, sequenceid=73, compaction requested=false 2024-11-21T03:41:01,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:01,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:01,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-21T03:41:01,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-21T03:41:01,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-21T03:41:01,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6190 sec 2024-11-21T03:41:01,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.6240 sec 2024-11-21T03:41:01,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:41:01,869 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:41:01,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17b90c50c9bd5dda59fffca9821d39a4 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T03:41:01,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/15bf8ecdfddd40fd894943a7f708faca is 50, key is test_row_0/A:col10/1732160461868/Put/seqid=0 2024-11-21T03:41:01,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742090_1266 (size=16681) 2024-11-21T03:41:01,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:01,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160521906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:01,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:01,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:01,911 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:01,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160521911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:01,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:01,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:01,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T03:41:02,017 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:41:02,017 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:41:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-21T03:41:02,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T03:41:02,019 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:41:02,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:02,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160522017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:02,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:02,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:02,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:02,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160522017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:02,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:02,020 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:41:02,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:02,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:02,020 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:41:02,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T03:41:02,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T03:41:02,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:02,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:02,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:02,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:02,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:02,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160522227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:02,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:02,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160522227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:02,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:02,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:02,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:02,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:02,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:02,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/15bf8ecdfddd40fd894943a7f708faca 2024-11-21T03:41:02,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/0b4e97f908b74239a55b5b9ab6af1ad7 is 50, key is test_row_0/B:col10/1732160461868/Put/seqid=0 2024-11-21T03:41:02,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742091_1267 (size=12001) 2024-11-21T03:41:02,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T03:41:02,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:02,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:02,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:02,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:02,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:02,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T03:41:02,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T03:41:02,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:02,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:02,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:02,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:02,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:02,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:02,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:02,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160522537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:02,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:02,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:02,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:02,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:02,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160522537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:02,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:02,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:02,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:02,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T03:41:02,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:02,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:02,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:02,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:02,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:02,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:02,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T03:41:02,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/0b4e97f908b74239a55b5b9ab6af1ad7 2024-11-21T03:41:02,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/545e3696f5a643e3987e6d548e2cf6d4 is 50, key is test_row_0/C:col10/1732160461868/Put/seqid=0 2024-11-21T03:41:02,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742092_1268 (size=12001) 2024-11-21T03:41:02,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T03:41:02,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:02,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:02,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:02,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:02,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:02,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:02,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T03:41:02,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:02,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:02,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:02,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:02,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:02,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:03,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160523047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:03,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:03,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:03,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:03,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160523047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:03,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:03,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:03,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:03,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:03,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:03,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50890 deadline: 1732160523086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:03,087 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_2' in 17b90c50c9bd5dda59fffca9821d39a4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:41:03,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:03,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:03,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:03,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T03:41:03,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:03,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:03,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:03,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:03,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:03,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:03,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50918 deadline: 1732160523097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:03,099 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_0' in 17b90c50c9bd5dda59fffca9821d39a4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:41:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:03,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:03,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732160523107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:03,109 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_1' in 17b90c50c9bd5dda59fffca9821d39a4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:41:03,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:03,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:03,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:03,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/545e3696f5a643e3987e6d548e2cf6d4 2024-11-21T03:41:03,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/15bf8ecdfddd40fd894943a7f708faca as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/15bf8ecdfddd40fd894943a7f708faca 2024-11-21T03:41:03,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/15bf8ecdfddd40fd894943a7f708faca, entries=250, sequenceid=91, filesize=16.3 K 2024-11-21T03:41:03,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/0b4e97f908b74239a55b5b9ab6af1ad7 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/0b4e97f908b74239a55b5b9ab6af1ad7 2024-11-21T03:41:03,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/0b4e97f908b74239a55b5b9ab6af1ad7, entries=150, sequenceid=91, filesize=11.7 K 2024-11-21T03:41:03,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/545e3696f5a643e3987e6d548e2cf6d4 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/545e3696f5a643e3987e6d548e2cf6d4 2024-11-21T03:41:03,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/545e3696f5a643e3987e6d548e2cf6d4, entries=150, sequenceid=91, filesize=11.7 K 2024-11-21T03:41:03,151 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 17b90c50c9bd5dda59fffca9821d39a4 in 1281ms, sequenceid=91, compaction requested=true 2024-11-21T03:41:03,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:03,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:41:03,151 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:03,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:03,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:41:03,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:03,151 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:03,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:41:03,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:03,152 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:03,152 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:03,152 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/B is initiating minor compaction (all files) 2024-11-21T03:41:03,152 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/A is initiating minor compaction (all files) 2024-11-21T03:41:03,152 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/A in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:03,152 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/B in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:03,152 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/c4245b08c7654752a0d8f182b9524903, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/d42af8e559a54d998bec7b6fea0d0bc3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/0b4e97f908b74239a55b5b9ab6af1ad7] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=35.3 K 2024-11-21T03:41:03,152 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/d663966aa82a4c2fba608e5f9881ffb1, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/a3165b39efeb47498e8a3965aa0517d6, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/15bf8ecdfddd40fd894943a7f708faca] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=39.8 K 2024-11-21T03:41:03,152 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d663966aa82a4c2fba608e5f9881ffb1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732160458929 2024-11-21T03:41:03,152 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting c4245b08c7654752a0d8f182b9524903, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732160458929 2024-11-21T03:41:03,153 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting d42af8e559a54d998bec7b6fea0d0bc3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732160460063 2024-11-21T03:41:03,153 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3165b39efeb47498e8a3965aa0517d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732160460063 2024-11-21T03:41:03,153 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15bf8ecdfddd40fd894943a7f708faca, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732160460720 2024-11-21T03:41:03,153 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b4e97f908b74239a55b5b9ab6af1ad7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732160460720 2024-11-21T03:41:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T03:41:03,161 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#A#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:03,161 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/1aa53261681d4c73825ad32690e48c70 is 50, key is test_row_0/A:col10/1732160461868/Put/seqid=0 2024-11-21T03:41:03,163 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#B#compaction#223 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:03,163 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/2c44bc16fd524c1ba34badc651cbd3ad is 50, key is test_row_0/B:col10/1732160461868/Put/seqid=0 2024-11-21T03:41:03,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742093_1269 (size=12207) 2024-11-21T03:41:03,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742094_1270 (size=12207) 2024-11-21T03:41:03,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T03:41:03,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:03,242 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 17b90c50c9bd5dda59fffca9821d39a4 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T03:41:03,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/6067e730df0e4964a996bbf967094f19 is 50, key is test_row_0/A:col10/1732160461895/Put/seqid=0 2024-11-21T03:41:03,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742095_1271 (size=12001) 2024-11-21T03:41:03,586 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/1aa53261681d4c73825ad32690e48c70 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/1aa53261681d4c73825ad32690e48c70 2024-11-21T03:41:03,590 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/2c44bc16fd524c1ba34badc651cbd3ad as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/2c44bc16fd524c1ba34badc651cbd3ad 2024-11-21T03:41:03,595 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/B of 17b90c50c9bd5dda59fffca9821d39a4 into 2c44bc16fd524c1ba34badc651cbd3ad(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:03,595 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:03,595 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/B, priority=13, startTime=1732160463151; duration=0sec 2024-11-21T03:41:03,595 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:03,595 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:B 2024-11-21T03:41:03,595 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:03,596 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:03,596 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/C is initiating minor compaction (all files) 2024-11-21T03:41:03,596 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/C in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:03,597 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/5f15c7cc40c14033a8a44617bb9959b9, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/ebd2ac6e8661449c885f3b19e3eb1439, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/545e3696f5a643e3987e6d548e2cf6d4] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=35.3 K 2024-11-21T03:41:03,597 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f15c7cc40c14033a8a44617bb9959b9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732160458929 2024-11-21T03:41:03,598 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting ebd2ac6e8661449c885f3b19e3eb1439, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1732160460063 2024-11-21T03:41:03,598 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 545e3696f5a643e3987e6d548e2cf6d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732160460720 2024-11-21T03:41:03,600 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/A of 17b90c50c9bd5dda59fffca9821d39a4 into 1aa53261681d4c73825ad32690e48c70(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:03,600 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:03,600 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/A, priority=13, startTime=1732160463151; duration=0sec 2024-11-21T03:41:03,600 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:03,600 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:A 2024-11-21T03:41:03,609 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#C#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:03,610 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/78b8f35be62b43daaaf59c0d40a7f59a is 50, key is test_row_0/C:col10/1732160461868/Put/seqid=0 2024-11-21T03:41:03,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742096_1272 (size=12207) 2024-11-21T03:41:03,632 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/78b8f35be62b43daaaf59c0d40a7f59a as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/78b8f35be62b43daaaf59c0d40a7f59a 2024-11-21T03:41:03,637 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/C of 17b90c50c9bd5dda59fffca9821d39a4 into 78b8f35be62b43daaaf59c0d40a7f59a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:03,637 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:03,637 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/C, priority=13, startTime=1732160463151; duration=0sec 2024-11-21T03:41:03,637 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:03,637 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:C 2024-11-21T03:41:03,650 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/6067e730df0e4964a996bbf967094f19 2024-11-21T03:41:03,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/d1b2f9cf850a459eb9eef3d94c893050 is 50, key is test_row_0/B:col10/1732160461895/Put/seqid=0 2024-11-21T03:41:03,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742097_1273 (size=12001) 2024-11-21T03:41:04,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:41:04,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:04,065 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/d1b2f9cf850a459eb9eef3d94c893050 2024-11-21T03:41:04,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/4cfcc5d8342a42aea90b09d3c2f68c99 is 50, key is test_row_0/C:col10/1732160461895/Put/seqid=0 2024-11-21T03:41:04,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742098_1274 (size=12001) 2024-11-21T03:41:04,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:04,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160524089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:04,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:04,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:04,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160524092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:04,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:04,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T03:41:04,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:04,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160524197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:04,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:04,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:04,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:04,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:04,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160524207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:04,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:04,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:04,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:04,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:04,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160524407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:04,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:04,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:04,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:04,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:04,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160524417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:04,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:04,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:04,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:04,492 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/4cfcc5d8342a42aea90b09d3c2f68c99 2024-11-21T03:41:04,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/6067e730df0e4964a996bbf967094f19 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/6067e730df0e4964a996bbf967094f19 2024-11-21T03:41:04,501 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/6067e730df0e4964a996bbf967094f19, entries=150, sequenceid=112, filesize=11.7 K 2024-11-21T03:41:04,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/d1b2f9cf850a459eb9eef3d94c893050 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/d1b2f9cf850a459eb9eef3d94c893050 2024-11-21T03:41:04,506 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/d1b2f9cf850a459eb9eef3d94c893050, entries=150, sequenceid=112, filesize=11.7 K 2024-11-21T03:41:04,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/4cfcc5d8342a42aea90b09d3c2f68c99 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/4cfcc5d8342a42aea90b09d3c2f68c99 2024-11-21T03:41:04,512 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/4cfcc5d8342a42aea90b09d3c2f68c99, entries=150, sequenceid=112, filesize=11.7 K 2024-11-21T03:41:04,513 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 17b90c50c9bd5dda59fffca9821d39a4 in 1271ms, sequenceid=112, compaction requested=false 2024-11-21T03:41:04,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:04,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:04,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-21T03:41:04,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-21T03:41:04,516 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-21T03:41:04,516 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4940 sec 2024-11-21T03:41:04,517 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.4990 sec 2024-11-21T03:41:04,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:41:04,719 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:41:04,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17b90c50c9bd5dda59fffca9821d39a4 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-21T03:41:04,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/39d50d4c0c6849cebd5ced4d790d0d45 is 50, key is test_row_0/A:col10/1732160464718/Put/seqid=0 2024-11-21T03:41:04,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742099_1275 (size=14441) 2024-11-21T03:41:04,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:04,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160524753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:04,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:04,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:04,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:04,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:04,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160524755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:04,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:04,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:04,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:04,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:04,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160524867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:04,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:04,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:04,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:04,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160524867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:04,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:04,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:04,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:04,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:05,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:05,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160525076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:05,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:05,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:05,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:05,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:05,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160525076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:05,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:05,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:05,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:05,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/39d50d4c0c6849cebd5ced4d790d0d45 2024-11-21T03:41:05,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/7a48199b4b9d46cfbc660aa0706dcede is 50, key is test_row_0/B:col10/1732160464718/Put/seqid=0 2024-11-21T03:41:05,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742100_1276 (size=12101) 2024-11-21T03:41:05,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:05,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160525387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:05,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:05,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:05,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:05,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:05,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160525387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:05,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:05,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:05,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:05,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/7a48199b4b9d46cfbc660aa0706dcede 2024-11-21T03:41:05,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/cd5abd61b862448db6312dce70caccb1 is 50, key is test_row_0/C:col10/1732160464718/Put/seqid=0 2024-11-21T03:41:05,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742101_1277 (size=12101) 2024-11-21T03:41:05,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160525897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:05,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160525897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:05,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:05,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/cd5abd61b862448db6312dce70caccb1 2024-11-21T03:41:05,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/39d50d4c0c6849cebd5ced4d790d0d45 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/39d50d4c0c6849cebd5ced4d790d0d45 2024-11-21T03:41:05,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/39d50d4c0c6849cebd5ced4d790d0d45, entries=200, sequenceid=131, filesize=14.1 K 2024-11-21T03:41:05,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/7a48199b4b9d46cfbc660aa0706dcede as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/7a48199b4b9d46cfbc660aa0706dcede 2024-11-21T03:41:06,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/7a48199b4b9d46cfbc660aa0706dcede, entries=150, sequenceid=131, filesize=11.8 K 2024-11-21T03:41:06,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/cd5abd61b862448db6312dce70caccb1 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/cd5abd61b862448db6312dce70caccb1 2024-11-21T03:41:06,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/cd5abd61b862448db6312dce70caccb1, entries=150, sequenceid=131, filesize=11.8 K 2024-11-21T03:41:06,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 17b90c50c9bd5dda59fffca9821d39a4 in 1291ms, sequenceid=131, compaction requested=true 2024-11-21T03:41:06,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:06,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:41:06,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:06,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:41:06,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:06,011 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:06,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:41:06,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T03:41:06,011 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:06,012 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38649 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:06,012 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/A is initiating minor compaction (all files) 2024-11-21T03:41:06,012 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/A in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:06,012 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/1aa53261681d4c73825ad32690e48c70, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/6067e730df0e4964a996bbf967094f19, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/39d50d4c0c6849cebd5ced4d790d0d45] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=37.7 K 2024-11-21T03:41:06,013 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:06,013 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/B is initiating minor compaction (all files) 2024-11-21T03:41:06,013 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/B in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:06,013 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/2c44bc16fd524c1ba34badc651cbd3ad, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/d1b2f9cf850a459eb9eef3d94c893050, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/7a48199b4b9d46cfbc660aa0706dcede] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=35.5 K 2024-11-21T03:41:06,013 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 1aa53261681d4c73825ad32690e48c70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732160460720 2024-11-21T03:41:06,014 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c44bc16fd524c1ba34badc651cbd3ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732160460720 2024-11-21T03:41:06,014 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 6067e730df0e4964a996bbf967094f19, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732160461889 2024-11-21T03:41:06,015 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1b2f9cf850a459eb9eef3d94c893050, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732160461889 2024-11-21T03:41:06,015 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 39d50d4c0c6849cebd5ced4d790d0d45, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732160464072 2024-11-21T03:41:06,016 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a48199b4b9d46cfbc660aa0706dcede, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732160464072 2024-11-21T03:41:06,049 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#A#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:06,050 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/421bec5b493748c397f68a7f34669e14 is 50, key is test_row_0/A:col10/1732160464718/Put/seqid=0 2024-11-21T03:41:06,071 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#B#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:06,072 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/36bc9cf6e75f466da56fd339c4e53297 is 50, key is test_row_0/B:col10/1732160464718/Put/seqid=0 2024-11-21T03:41:06,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742102_1278 (size=12409) 2024-11-21T03:41:06,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742103_1279 (size=12409) 2024-11-21T03:41:06,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T03:41:06,177 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:41:06,178 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:41:06,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-21T03:41:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T03:41:06,180 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:41:06,181 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:41:06,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:41:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T03:41:06,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T03:41:06,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:06,333 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 17b90c50c9bd5dda59fffca9821d39a4 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-21T03:41:06,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/e708952d49bd4e0a94fea638baeabfbc is 50, key is test_row_0/A:col10/1732160464751/Put/seqid=0 2024-11-21T03:41:06,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742104_1280 (size=12151) 2024-11-21T03:41:06,365 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/e708952d49bd4e0a94fea638baeabfbc 2024-11-21T03:41:06,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/933f62c30b7c43d09fdae28dabb0f672 is 50, key is test_row_0/B:col10/1732160464751/Put/seqid=0 2024-11-21T03:41:06,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742105_1281 (size=12151) 2024-11-21T03:41:06,467 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-21T03:41:06,467 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-21T03:41:06,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T03:41:06,503 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/36bc9cf6e75f466da56fd339c4e53297 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/36bc9cf6e75f466da56fd339c4e53297 2024-11-21T03:41:06,506 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/421bec5b493748c397f68a7f34669e14 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/421bec5b493748c397f68a7f34669e14 2024-11-21T03:41:06,512 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/B of 17b90c50c9bd5dda59fffca9821d39a4 into 36bc9cf6e75f466da56fd339c4e53297(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:06,512 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:06,512 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/B, priority=13, startTime=1732160466011; duration=0sec 2024-11-21T03:41:06,512 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:06,512 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:B 2024-11-21T03:41:06,512 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:06,517 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:06,517 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/C is initiating minor compaction (all files) 2024-11-21T03:41:06,517 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/C in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:06,517 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/78b8f35be62b43daaaf59c0d40a7f59a, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/4cfcc5d8342a42aea90b09d3c2f68c99, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/cd5abd61b862448db6312dce70caccb1] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=35.5 K 2024-11-21T03:41:06,518 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/A of 17b90c50c9bd5dda59fffca9821d39a4 into 421bec5b493748c397f68a7f34669e14(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:06,518 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:06,518 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/A, priority=13, startTime=1732160466011; duration=0sec 2024-11-21T03:41:06,518 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:06,518 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:A 2024-11-21T03:41:06,522 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78b8f35be62b43daaaf59c0d40a7f59a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732160460720 2024-11-21T03:41:06,523 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cfcc5d8342a42aea90b09d3c2f68c99, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732160461889 2024-11-21T03:41:06,524 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd5abd61b862448db6312dce70caccb1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732160464072 2024-11-21T03:41:06,532 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#C#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:06,532 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/8b250df71c7a40c8867819998d54d39f is 50, key is test_row_0/C:col10/1732160464718/Put/seqid=0 2024-11-21T03:41:06,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742106_1282 (size=12409) 2024-11-21T03:41:06,589 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/8b250df71c7a40c8867819998d54d39f as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/8b250df71c7a40c8867819998d54d39f 2024-11-21T03:41:06,598 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/C of 17b90c50c9bd5dda59fffca9821d39a4 into 8b250df71c7a40c8867819998d54d39f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:06,598 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:06,598 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/C, priority=13, startTime=1732160466011; duration=0sec 2024-11-21T03:41:06,598 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:06,598 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:C 2024-11-21T03:41:06,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T03:41:06,817 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/933f62c30b7c43d09fdae28dabb0f672 2024-11-21T03:41:06,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/2f4bba879c5a4df5acfd43c92f3d2577 is 50, key is test_row_0/C:col10/1732160464751/Put/seqid=0 2024-11-21T03:41:06,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742107_1283 (size=12151) 2024-11-21T03:41:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:41:06,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:06,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160526944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:06,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160526944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:06,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:06,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:06,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:06,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:06,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:07,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160527057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:07,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:07,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:07,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160527057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:07,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:07,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:07,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:07,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:07,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:07,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50890 deadline: 1732160527106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:07,110 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_2' in 17b90c50c9bd5dda59fffca9821d39a4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:41:07,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:07,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:07,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:07,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50918 deadline: 1732160527127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:07,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732160527127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:07,129 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_0' in 17b90c50c9bd5dda59fffca9821d39a4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:41:07,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:07,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:07,130 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_1' in 17b90c50c9bd5dda59fffca9821d39a4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:41:07,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:07,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:07,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:07,130 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:07,242 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/2f4bba879c5a4df5acfd43c92f3d2577 2024-11-21T03:41:07,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/e708952d49bd4e0a94fea638baeabfbc as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/e708952d49bd4e0a94fea638baeabfbc 2024-11-21T03:41:07,250 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/e708952d49bd4e0a94fea638baeabfbc, entries=150, sequenceid=151, filesize=11.9 K 2024-11-21T03:41:07,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/933f62c30b7c43d09fdae28dabb0f672 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/933f62c30b7c43d09fdae28dabb0f672 2024-11-21T03:41:07,255 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/933f62c30b7c43d09fdae28dabb0f672, entries=150, sequenceid=151, filesize=11.9 K 2024-11-21T03:41:07,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/2f4bba879c5a4df5acfd43c92f3d2577 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/2f4bba879c5a4df5acfd43c92f3d2577 2024-11-21T03:41:07,259 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/2f4bba879c5a4df5acfd43c92f3d2577, entries=150, sequenceid=151, filesize=11.9 K 2024-11-21T03:41:07,260 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 17b90c50c9bd5dda59fffca9821d39a4 in 927ms, sequenceid=151, compaction requested=false 2024-11-21T03:41:07,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:07,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:07,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-21T03:41:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-21T03:41:07,263 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-21T03:41:07,263 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0800 sec 2024-11-21T03:41:07,265 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.0850 sec 2024-11-21T03:41:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:41:07,269 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:41:07,269 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17b90c50c9bd5dda59fffca9821d39a4 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-21T03:41:07,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/00b0b2a70dc241728556eb39622ec120 is 50, key is test_row_0/A:col10/1732160466939/Put/seqid=0 2024-11-21T03:41:07,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742108_1284 (size=12151) 2024-11-21T03:41:07,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/00b0b2a70dc241728556eb39622ec120 2024-11-21T03:41:07,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/98d6c7fc9f754594b911a5d681a29b12 is 50, key is test_row_0/B:col10/1732160466939/Put/seqid=0 2024-11-21T03:41:07,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160527307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:07,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160527307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:07,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:07,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:07,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:07,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:07,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:07,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:07,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742109_1285 (size=12151) 2024-11-21T03:41:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T03:41:07,327 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:41:07,328 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:41:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-21T03:41:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T03:41:07,331 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:41:07,332 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:41:07,332 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:41:07,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160527417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:07,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160527417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:07,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T03:41:07,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T03:41:07,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:07,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:07,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:07,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:07,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:07,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:07,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160527626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:07,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:07,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:07,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:07,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:07,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160527626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:07,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:07,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:07,630 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:07,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T03:41:07,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:07,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:07,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:07,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:07,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:07,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:07,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T03:41:07,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/98d6c7fc9f754594b911a5d681a29b12 2024-11-21T03:41:07,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/5481b94b6912484fa6ca71d0fc8ab6df is 50, key is test_row_0/C:col10/1732160466939/Put/seqid=0 2024-11-21T03:41:07,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742110_1286 (size=12151) 2024-11-21T03:41:07,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T03:41:07,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:07,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:07,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:07,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:07,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:07,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:07,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:07,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160527937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:07,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:07,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:07,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:07,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:07,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160527937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:07,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:07,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:07,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:07,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T03:41:07,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:07,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:07,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:07,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:07,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:07,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:07,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T03:41:08,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T03:41:08,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:08,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:08,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:08,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:08,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:08,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/5481b94b6912484fa6ca71d0fc8ab6df 2024-11-21T03:41:08,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/00b0b2a70dc241728556eb39622ec120 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/00b0b2a70dc241728556eb39622ec120 2024-11-21T03:41:08,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/00b0b2a70dc241728556eb39622ec120, entries=150, sequenceid=173, filesize=11.9 K 2024-11-21T03:41:08,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/98d6c7fc9f754594b911a5d681a29b12 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/98d6c7fc9f754594b911a5d681a29b12 2024-11-21T03:41:08,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/98d6c7fc9f754594b911a5d681a29b12, entries=150, sequenceid=173, filesize=11.9 K 2024-11-21T03:41:08,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/5481b94b6912484fa6ca71d0fc8ab6df as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/5481b94b6912484fa6ca71d0fc8ab6df 2024-11-21T03:41:08,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/5481b94b6912484fa6ca71d0fc8ab6df, entries=150, sequenceid=173, filesize=11.9 K 2024-11-21T03:41:08,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 17b90c50c9bd5dda59fffca9821d39a4 in 880ms, sequenceid=173, compaction requested=true 2024-11-21T03:41:08,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:08,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:41:08,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:08,150 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:08,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:41:08,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:08,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:41:08,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T03:41:08,150 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:08,151 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:08,151 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/A is initiating minor compaction (all files) 2024-11-21T03:41:08,151 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/A in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:08,151 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:08,151 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/421bec5b493748c397f68a7f34669e14, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/e708952d49bd4e0a94fea638baeabfbc, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/00b0b2a70dc241728556eb39622ec120] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=35.9 K 2024-11-21T03:41:08,151 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/B is initiating minor compaction (all files) 2024-11-21T03:41:08,151 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/B in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:08,151 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/36bc9cf6e75f466da56fd339c4e53297, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/933f62c30b7c43d09fdae28dabb0f672, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/98d6c7fc9f754594b911a5d681a29b12] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=35.9 K 2024-11-21T03:41:08,151 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 421bec5b493748c397f68a7f34669e14, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732160464072 2024-11-21T03:41:08,152 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36bc9cf6e75f466da56fd339c4e53297, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732160464072 2024-11-21T03:41:08,152 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting e708952d49bd4e0a94fea638baeabfbc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732160464750 2024-11-21T03:41:08,152 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 933f62c30b7c43d09fdae28dabb0f672, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732160464750 2024-11-21T03:41:08,152 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 00b0b2a70dc241728556eb39622ec120, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732160466939 2024-11-21T03:41:08,152 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98d6c7fc9f754594b911a5d681a29b12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732160466939 2024-11-21T03:41:08,159 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#B#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:08,160 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/0900ca2ae3984d4586ff31a0df434472 is 50, key is test_row_0/B:col10/1732160466939/Put/seqid=0 2024-11-21T03:41:08,165 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#A#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:08,165 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/a707374d21304af693b346930ca55464 is 50, key is test_row_0/A:col10/1732160466939/Put/seqid=0 2024-11-21T03:41:08,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742111_1287 (size=12561) 2024-11-21T03:41:08,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742112_1288 (size=12561) 2024-11-21T03:41:08,191 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/0900ca2ae3984d4586ff31a0df434472 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/0900ca2ae3984d4586ff31a0df434472 2024-11-21T03:41:08,192 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/a707374d21304af693b346930ca55464 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/a707374d21304af693b346930ca55464 2024-11-21T03:41:08,199 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/A of 17b90c50c9bd5dda59fffca9821d39a4 into a707374d21304af693b346930ca55464(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:08,199 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:08,199 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/A, priority=13, startTime=1732160468149; duration=0sec 2024-11-21T03:41:08,199 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:08,199 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:A 2024-11-21T03:41:08,199 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:08,199 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/B of 17b90c50c9bd5dda59fffca9821d39a4 into 0900ca2ae3984d4586ff31a0df434472(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:08,200 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:08,200 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/B, priority=13, startTime=1732160468150; duration=0sec 2024-11-21T03:41:08,200 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:08,200 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:B 2024-11-21T03:41:08,201 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:08,201 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/C is initiating minor compaction (all files) 2024-11-21T03:41:08,201 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/C in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:08,201 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/8b250df71c7a40c8867819998d54d39f, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/2f4bba879c5a4df5acfd43c92f3d2577, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/5481b94b6912484fa6ca71d0fc8ab6df] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=35.9 K 2024-11-21T03:41:08,201 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b250df71c7a40c8867819998d54d39f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732160464072 2024-11-21T03:41:08,202 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f4bba879c5a4df5acfd43c92f3d2577, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1732160464750 2024-11-21T03:41:08,202 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 5481b94b6912484fa6ca71d0fc8ab6df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732160466939 2024-11-21T03:41:08,211 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#C#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:08,211 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/ce8e72186ada4ed997c6db144ed54126 is 50, key is test_row_0/C:col10/1732160466939/Put/seqid=0 2024-11-21T03:41:08,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742113_1289 (size=12561) 2024-11-21T03:41:08,222 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/ce8e72186ada4ed997c6db144ed54126 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/ce8e72186ada4ed997c6db144ed54126 2024-11-21T03:41:08,228 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/C of 17b90c50c9bd5dda59fffca9821d39a4 into ce8e72186ada4ed997c6db144ed54126(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:08,228 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:08,228 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/C, priority=13, startTime=1732160468150; duration=0sec 2024-11-21T03:41:08,228 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:08,228 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:C 2024-11-21T03:41:08,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T03:41:08,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:08,248 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 17b90c50c9bd5dda59fffca9821d39a4 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-21T03:41:08,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/925eb946497141db93d01d9561c799e1 is 50, key is test_row_0/A:col10/1732160467302/Put/seqid=0 2024-11-21T03:41:08,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742114_1290 (size=12151) 2024-11-21T03:41:08,261 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/925eb946497141db93d01d9561c799e1 2024-11-21T03:41:08,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/f9b6d71f5031460f8cc81d7b770873f0 is 50, key is test_row_0/B:col10/1732160467302/Put/seqid=0 2024-11-21T03:41:08,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742115_1291 (size=12151) 2024-11-21T03:41:08,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:41:08,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:08,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T03:41:08,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:08,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160528505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:08,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:08,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160528505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:08,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160528617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:08,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:08,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:08,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:08,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:08,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160528617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:08,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:08,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:08,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:08,675 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/f9b6d71f5031460f8cc81d7b770873f0 2024-11-21T03:41:08,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/810c5394dada4367a399bb7dc4a314c1 is 50, key is test_row_0/C:col10/1732160467302/Put/seqid=0 2024-11-21T03:41:08,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742116_1292 (size=12151) 2024-11-21T03:41:08,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:08,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160528826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:08,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:08,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160528827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:08,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:08,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:08,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:08,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:08,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:08,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:09,087 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/810c5394dada4367a399bb7dc4a314c1 2024-11-21T03:41:09,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/925eb946497141db93d01d9561c799e1 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/925eb946497141db93d01d9561c799e1 2024-11-21T03:41:09,099 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/925eb946497141db93d01d9561c799e1, entries=150, sequenceid=193, filesize=11.9 K 2024-11-21T03:41:09,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/f9b6d71f5031460f8cc81d7b770873f0 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/f9b6d71f5031460f8cc81d7b770873f0 2024-11-21T03:41:09,106 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/f9b6d71f5031460f8cc81d7b770873f0, entries=150, sequenceid=193, filesize=11.9 K 2024-11-21T03:41:09,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/810c5394dada4367a399bb7dc4a314c1 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/810c5394dada4367a399bb7dc4a314c1 2024-11-21T03:41:09,112 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/810c5394dada4367a399bb7dc4a314c1, entries=150, sequenceid=193, filesize=11.9 K 2024-11-21T03:41:09,113 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 17b90c50c9bd5dda59fffca9821d39a4 in 865ms, sequenceid=193, compaction requested=false 2024-11-21T03:41:09,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:09,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:09,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-21T03:41:09,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-21T03:41:09,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-21T03:41:09,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7820 sec 2024-11-21T03:41:09,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.7890 sec 2024-11-21T03:41:09,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:41:09,140 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:41:09,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17b90c50c9bd5dda59fffca9821d39a4 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-21T03:41:09,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/45705b7154194020960bd277ad28ed7a is 50, key is test_row_0/A:col10/1732160469140/Put/seqid=0 2024-11-21T03:41:09,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742117_1293 (size=14541) 2024-11-21T03:41:09,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160529164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:09,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:09,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:09,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:09,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:09,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160529168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:09,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:09,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:09,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:09,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160529277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:09,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:09,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160529277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:09,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:09,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:09,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:09,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:09,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:09,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T03:41:09,477 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:41:09,478 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:41:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-21T03:41:09,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T03:41:09,479 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:41:09,480 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:41:09,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:41:09,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:09,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160529487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:09,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:09,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:09,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160529487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:09,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:09,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:09,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:09,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:09,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/45705b7154194020960bd277ad28ed7a 2024-11-21T03:41:09,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/09613b6a77fa4ed6bcaacbc2a5d5d4ca is 50, key is test_row_0/B:col10/1732160469140/Put/seqid=0 2024-11-21T03:41:09,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742118_1294 (size=12151) 2024-11-21T03:41:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T03:41:09,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-21T03:41:09,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:09,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:09,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:09,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:09,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:09,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-21T03:41:09,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:09,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:09,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:09,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:09,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:09,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:09,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T03:41:09,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:09,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:09,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160529797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:09,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160529797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:09,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:09,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-21T03:41:09,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:09,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:09,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:09,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:09,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:09,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:09,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/09613b6a77fa4ed6bcaacbc2a5d5d4ca 2024-11-21T03:41:09,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/270766ef70ae472388cbc574290f886a is 50, key is test_row_0/C:col10/1732160469140/Put/seqid=0 2024-11-21T03:41:09,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742119_1295 (size=12151) 2024-11-21T03:41:09,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/270766ef70ae472388cbc574290f886a 2024-11-21T03:41:09,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/45705b7154194020960bd277ad28ed7a as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/45705b7154194020960bd277ad28ed7a 2024-11-21T03:41:09,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/45705b7154194020960bd277ad28ed7a, entries=200, sequenceid=214, filesize=14.2 K 2024-11-21T03:41:09,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/09613b6a77fa4ed6bcaacbc2a5d5d4ca as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/09613b6a77fa4ed6bcaacbc2a5d5d4ca 2024-11-21T03:41:09,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/09613b6a77fa4ed6bcaacbc2a5d5d4ca, entries=150, sequenceid=214, filesize=11.9 K 2024-11-21T03:41:09,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/270766ef70ae472388cbc574290f886a as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/270766ef70ae472388cbc574290f886a 2024-11-21T03:41:09,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/270766ef70ae472388cbc574290f886a, entries=150, sequenceid=214, filesize=11.9 K 2024-11-21T03:41:09,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 17b90c50c9bd5dda59fffca9821d39a4 in 854ms, sequenceid=214, compaction requested=true 2024-11-21T03:41:09,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:09,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:41:09,994 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:09,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:09,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:41:09,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:09,994 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:09,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:41:09,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:09,995 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:09,995 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/A is initiating minor compaction (all files) 2024-11-21T03:41:09,995 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:09,996 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/B is initiating minor compaction (all files) 2024-11-21T03:41:09,996 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/A in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:09,996 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/B in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:09,996 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/0900ca2ae3984d4586ff31a0df434472, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/f9b6d71f5031460f8cc81d7b770873f0, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/09613b6a77fa4ed6bcaacbc2a5d5d4ca] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=36.0 K 2024-11-21T03:41:09,996 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/a707374d21304af693b346930ca55464, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/925eb946497141db93d01d9561c799e1, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/45705b7154194020960bd277ad28ed7a] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=38.3 K 2024-11-21T03:41:09,996 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0900ca2ae3984d4586ff31a0df434472, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732160466939 2024-11-21T03:41:09,996 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting a707374d21304af693b346930ca55464, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732160466939 2024-11-21T03:41:09,996 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting f9b6d71f5031460f8cc81d7b770873f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732160467294 2024-11-21T03:41:09,996 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 925eb946497141db93d01d9561c799e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732160467294 2024-11-21T03:41:09,996 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 09613b6a77fa4ed6bcaacbc2a5d5d4ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732160468474 2024-11-21T03:41:09,996 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45705b7154194020960bd277ad28ed7a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732160468474 2024-11-21T03:41:10,003 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#B#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:10,003 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/9c2ad4bdf06b4a79a1ea648b2a350878 is 50, key is test_row_0/B:col10/1732160469140/Put/seqid=0 2024-11-21T03:41:10,004 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#A#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:10,004 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/d4097266e16844be883425fc50ad5cad is 50, key is test_row_0/A:col10/1732160469140/Put/seqid=0 2024-11-21T03:41:10,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742120_1296 (size=12663) 2024-11-21T03:41:10,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742121_1297 (size=12663) 2024-11-21T03:41:10,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-21T03:41:10,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:10,090 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 17b90c50c9bd5dda59fffca9821d39a4 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T03:41:10,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/37dc7c51b515467091fb377be7f9540f is 50, key is test_row_0/A:col10/1732160469167/Put/seqid=0 2024-11-21T03:41:10,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T03:41:10,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742122_1298 (size=12151) 2024-11-21T03:41:10,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:41:10,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:10,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:10,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160530342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:10,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:10,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:10,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160530342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:10,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:10,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:10,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:10,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:10,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:10,419 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/9c2ad4bdf06b4a79a1ea648b2a350878 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/9c2ad4bdf06b4a79a1ea648b2a350878 2024-11-21T03:41:10,423 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/B of 17b90c50c9bd5dda59fffca9821d39a4 into 9c2ad4bdf06b4a79a1ea648b2a350878(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:10,423 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:10,423 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/B, priority=13, startTime=1732160469994; duration=0sec 2024-11-21T03:41:10,423 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:10,423 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:B 2024-11-21T03:41:10,423 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:10,424 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:10,424 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/C is initiating minor compaction (all files) 2024-11-21T03:41:10,424 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/C in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:10,424 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/ce8e72186ada4ed997c6db144ed54126, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/810c5394dada4367a399bb7dc4a314c1, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/270766ef70ae472388cbc574290f886a] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=36.0 K 2024-11-21T03:41:10,425 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting ce8e72186ada4ed997c6db144ed54126, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732160466939 2024-11-21T03:41:10,426 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 810c5394dada4367a399bb7dc4a314c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732160467294 2024-11-21T03:41:10,426 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 270766ef70ae472388cbc574290f886a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732160468474 2024-11-21T03:41:10,427 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/d4097266e16844be883425fc50ad5cad as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/d4097266e16844be883425fc50ad5cad 2024-11-21T03:41:10,431 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/A of 17b90c50c9bd5dda59fffca9821d39a4 into d4097266e16844be883425fc50ad5cad(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:10,431 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:10,431 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/A, priority=13, startTime=1732160469994; duration=0sec 2024-11-21T03:41:10,431 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:10,431 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:A 2024-11-21T03:41:10,432 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#C#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:10,432 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/a1d94041d078465585092aea0e6a11a4 is 50, key is test_row_0/C:col10/1732160469140/Put/seqid=0 2024-11-21T03:41:10,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742123_1299 (size=12663) 2024-11-21T03:41:10,439 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/a1d94041d078465585092aea0e6a11a4 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/a1d94041d078465585092aea0e6a11a4 2024-11-21T03:41:10,443 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/C of 17b90c50c9bd5dda59fffca9821d39a4 into a1d94041d078465585092aea0e6a11a4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:10,443 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:10,443 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/C, priority=13, startTime=1732160469994; duration=0sec 2024-11-21T03:41:10,444 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:10,444 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:C 2024-11-21T03:41:10,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:10,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160530457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:10,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:10,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:10,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:10,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:10,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160530457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:10,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:10,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:10,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:10,513 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/37dc7c51b515467091fb377be7f9540f 2024-11-21T03:41:10,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/3bb6c88be6704b2990eddcd98a26da5d is 50, key is test_row_0/B:col10/1732160469167/Put/seqid=0 2024-11-21T03:41:10,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742124_1300 (size=12151) 2024-11-21T03:41:10,539 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/3bb6c88be6704b2990eddcd98a26da5d 2024-11-21T03:41:10,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/94492f1e5971457a846e274d1c0b36d2 is 50, key is test_row_0/C:col10/1732160469167/Put/seqid=0 2024-11-21T03:41:10,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742125_1301 (size=12151) 2024-11-21T03:41:10,564 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/94492f1e5971457a846e274d1c0b36d2 2024-11-21T03:41:10,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/37dc7c51b515467091fb377be7f9540f as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/37dc7c51b515467091fb377be7f9540f 2024-11-21T03:41:10,572 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/37dc7c51b515467091fb377be7f9540f, entries=150, sequenceid=229, filesize=11.9 K 2024-11-21T03:41:10,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/3bb6c88be6704b2990eddcd98a26da5d as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/3bb6c88be6704b2990eddcd98a26da5d 2024-11-21T03:41:10,577 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/3bb6c88be6704b2990eddcd98a26da5d, entries=150, sequenceid=229, filesize=11.9 K 2024-11-21T03:41:10,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/94492f1e5971457a846e274d1c0b36d2 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/94492f1e5971457a846e274d1c0b36d2 2024-11-21T03:41:10,583 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/94492f1e5971457a846e274d1c0b36d2, entries=150, sequenceid=229, filesize=11.9 K 2024-11-21T03:41:10,584 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 17b90c50c9bd5dda59fffca9821d39a4 in 494ms, sequenceid=229, compaction requested=false 2024-11-21T03:41:10,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:10,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:10,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-21T03:41:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-21T03:41:10,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-21T03:41:10,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1050 sec 2024-11-21T03:41:10,590 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.1100 sec 2024-11-21T03:41:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T03:41:10,616 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:41:10,617 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:41:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-21T03:41:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T03:41:10,618 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:41:10,619 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:41:10,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:41:10,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:41:10,670 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:41:10,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17b90c50c9bd5dda59fffca9821d39a4 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T03:41:10,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/076325d3eba845db86294fa5b2df0565 is 50, key is test_row_0/A:col10/1732160470668/Put/seqid=0 2024-11-21T03:41:10,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742126_1302 (size=12151) 2024-11-21T03:41:10,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/076325d3eba845db86294fa5b2df0565 2024-11-21T03:41:10,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/506088fcd7074c4a9d408df5b07c5dd5 is 50, key is test_row_0/B:col10/1732160470668/Put/seqid=0 2024-11-21T03:41:10,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742127_1303 (size=12151) 2024-11-21T03:41:10,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/506088fcd7074c4a9d408df5b07c5dd5 2024-11-21T03:41:10,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160530693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:10,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160530695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:10,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:10,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/46acc9d8ae7f470b93be56332e068576 is 50, key is test_row_0/C:col10/1732160470668/Put/seqid=0 2024-11-21T03:41:10,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742128_1304 (size=12151) 2024-11-21T03:41:10,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T03:41:10,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T03:41:10,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:10,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:10,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:10,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:10,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:10,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:10,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:10,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160530807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:10,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:10,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:10,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:10,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:10,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160530807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:10,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:10,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:10,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:10,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T03:41:10,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:10,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:10,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:10,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:10,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:10,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:10,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T03:41:11,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:11,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160531017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:11,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:11,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160531017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:11,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:11,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T03:41:11,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:11,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:11,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:11,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:11,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:11,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:11,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/46acc9d8ae7f470b93be56332e068576 2024-11-21T03:41:11,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/076325d3eba845db86294fa5b2df0565 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/076325d3eba845db86294fa5b2df0565 2024-11-21T03:41:11,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/076325d3eba845db86294fa5b2df0565, entries=150, sequenceid=255, filesize=11.9 K 2024-11-21T03:41:11,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/506088fcd7074c4a9d408df5b07c5dd5 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/506088fcd7074c4a9d408df5b07c5dd5 2024-11-21T03:41:11,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/506088fcd7074c4a9d408df5b07c5dd5, entries=150, sequenceid=255, filesize=11.9 K 2024-11-21T03:41:11,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/46acc9d8ae7f470b93be56332e068576 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/46acc9d8ae7f470b93be56332e068576 2024-11-21T03:41:11,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/46acc9d8ae7f470b93be56332e068576, entries=150, sequenceid=255, filesize=11.9 K 2024-11-21T03:41:11,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 17b90c50c9bd5dda59fffca9821d39a4 in 454ms, sequenceid=255, compaction requested=true 2024-11-21T03:41:11,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:11,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:41:11,124 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:11,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:11,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:41:11,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:11,124 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:11,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:41:11,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:11,125 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:11,125 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/A is initiating minor compaction (all files) 2024-11-21T03:41:11,125 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/A in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:11,125 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/d4097266e16844be883425fc50ad5cad, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/37dc7c51b515467091fb377be7f9540f, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/076325d3eba845db86294fa5b2df0565] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=36.1 K 2024-11-21T03:41:11,125 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:11,125 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/B is initiating minor compaction (all files) 2024-11-21T03:41:11,125 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/B in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:11,125 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4097266e16844be883425fc50ad5cad, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732160468474 2024-11-21T03:41:11,125 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/9c2ad4bdf06b4a79a1ea648b2a350878, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/3bb6c88be6704b2990eddcd98a26da5d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/506088fcd7074c4a9d408df5b07c5dd5] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=36.1 K 2024-11-21T03:41:11,126 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37dc7c51b515467091fb377be7f9540f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732160469162 2024-11-21T03:41:11,126 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c2ad4bdf06b4a79a1ea648b2a350878, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732160468474 2024-11-21T03:41:11,126 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bb6c88be6704b2990eddcd98a26da5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732160469162 2024-11-21T03:41:11,126 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 076325d3eba845db86294fa5b2df0565, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732160470340 2024-11-21T03:41:11,126 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 506088fcd7074c4a9d408df5b07c5dd5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732160470340 2024-11-21T03:41:11,132 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#A#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:11,133 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/507e14ee630849bcbf69e59653ec1db1 is 50, key is test_row_0/A:col10/1732160470668/Put/seqid=0 2024-11-21T03:41:11,133 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#B#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:11,134 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/f2b48261f33a4037ba935179ddc105d5 is 50, key is test_row_0/B:col10/1732160470668/Put/seqid=0 2024-11-21T03:41:11,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742129_1305 (size=12765) 2024-11-21T03:41:11,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742130_1306 (size=12765) 2024-11-21T03:41:11,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T03:41:11,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:11,230 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 17b90c50c9bd5dda59fffca9821d39a4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T03:41:11,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/f7731ee868f94e20be88468c7a21881b is 50, key is test_row_0/A:col10/1732160470691/Put/seqid=0 2024-11-21T03:41:11,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742131_1307 (size=12301) 2024-11-21T03:41:11,238 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/f7731ee868f94e20be88468c7a21881b 2024-11-21T03:41:11,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/2740747bfe1c40a684c1ae6d46d2e78b is 50, key is test_row_0/B:col10/1732160470691/Put/seqid=0 2024-11-21T03:41:11,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T03:41:11,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742132_1308 (size=12301) 2024-11-21T03:41:11,249 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/2740747bfe1c40a684c1ae6d46d2e78b 2024-11-21T03:41:11,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/9ef45713d45a4b788149545c2d0dfed9 is 50, key is test_row_0/C:col10/1732160470691/Put/seqid=0 2024-11-21T03:41:11,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742133_1309 (size=12301) 2024-11-21T03:41:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:41:11,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:11,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160531366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:11,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:11,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:11,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160531366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:11,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:11,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:11,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:11,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:11,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:11,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:11,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160531477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:11,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:11,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:11,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160531477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:11,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:11,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:11,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:11,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:11,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:11,553 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/f2b48261f33a4037ba935179ddc105d5 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/f2b48261f33a4037ba935179ddc105d5 2024-11-21T03:41:11,553 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/507e14ee630849bcbf69e59653ec1db1 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/507e14ee630849bcbf69e59653ec1db1 2024-11-21T03:41:11,557 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/A of 17b90c50c9bd5dda59fffca9821d39a4 into 507e14ee630849bcbf69e59653ec1db1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:11,557 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/B of 17b90c50c9bd5dda59fffca9821d39a4 into f2b48261f33a4037ba935179ddc105d5(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:11,557 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:11,557 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:11,557 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/A, priority=13, startTime=1732160471124; duration=0sec 2024-11-21T03:41:11,557 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/B, priority=13, startTime=1732160471124; duration=0sec 2024-11-21T03:41:11,557 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:11,557 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:A 2024-11-21T03:41:11,557 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:11,558 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:B 2024-11-21T03:41:11,558 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:11,558 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:11,559 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/C is initiating minor compaction (all files) 2024-11-21T03:41:11,559 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/C in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:11,559 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/a1d94041d078465585092aea0e6a11a4, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/94492f1e5971457a846e274d1c0b36d2, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/46acc9d8ae7f470b93be56332e068576] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=36.1 K 2024-11-21T03:41:11,559 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1d94041d078465585092aea0e6a11a4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732160468474 2024-11-21T03:41:11,559 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94492f1e5971457a846e274d1c0b36d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1732160469162 2024-11-21T03:41:11,560 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46acc9d8ae7f470b93be56332e068576, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732160470340 2024-11-21T03:41:11,565 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#C#compaction#263 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:11,565 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/c51c4282209445bbbec0b7ce132f4e8f is 50, key is test_row_0/C:col10/1732160470668/Put/seqid=0 2024-11-21T03:41:11,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742134_1310 (size=12765) 2024-11-21T03:41:11,573 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/c51c4282209445bbbec0b7ce132f4e8f as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/c51c4282209445bbbec0b7ce132f4e8f 2024-11-21T03:41:11,578 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/C of 17b90c50c9bd5dda59fffca9821d39a4 into c51c4282209445bbbec0b7ce132f4e8f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:11,578 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:11,578 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/C, priority=13, startTime=1732160471124; duration=0sec 2024-11-21T03:41:11,578 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:11,578 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:C 2024-11-21T03:41:11,659 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/9ef45713d45a4b788149545c2d0dfed9 2024-11-21T03:41:11,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/f7731ee868f94e20be88468c7a21881b as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/f7731ee868f94e20be88468c7a21881b 2024-11-21T03:41:11,674 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/f7731ee868f94e20be88468c7a21881b, entries=150, sequenceid=268, filesize=12.0 K 2024-11-21T03:41:11,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/2740747bfe1c40a684c1ae6d46d2e78b as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/2740747bfe1c40a684c1ae6d46d2e78b 2024-11-21T03:41:11,679 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/2740747bfe1c40a684c1ae6d46d2e78b, entries=150, sequenceid=268, filesize=12.0 K 2024-11-21T03:41:11,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/9ef45713d45a4b788149545c2d0dfed9 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/9ef45713d45a4b788149545c2d0dfed9 2024-11-21T03:41:11,686 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/9ef45713d45a4b788149545c2d0dfed9, entries=150, sequenceid=268, filesize=12.0 K 2024-11-21T03:41:11,687 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 17b90c50c9bd5dda59fffca9821d39a4 in 458ms, sequenceid=268, compaction requested=false 2024-11-21T03:41:11,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:11,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:11,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-21T03:41:11,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-21T03:41:11,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:41:11,690 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:41:11,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17b90c50c9bd5dda59fffca9821d39a4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T03:41:11,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-21T03:41:11,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0690 sec 2024-11-21T03:41:11,693 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.0750 sec 2024-11-21T03:41:11,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/6e22cca4cc2849a9bded90f4568d820b is 50, key is test_row_0/A:col10/1732160471365/Put/seqid=0 2024-11-21T03:41:11,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742135_1311 (size=14741) 2024-11-21T03:41:11,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/6e22cca4cc2849a9bded90f4568d820b 2024-11-21T03:41:11,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:11,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160531712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:11,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/c1bbfd6134ee4838bef41341c8e6166e is 50, key is test_row_0/B:col10/1732160471365/Put/seqid=0 2024-11-21T03:41:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:11,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:11,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:11,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160531716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:11,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742136_1312 (size=12301) 2024-11-21T03:41:11,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:11,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:11,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:11,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T03:41:11,757 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:41:11,757 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:41:11,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-21T03:41:11,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T03:41:11,759 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:41:11,760 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:41:11,760 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:41:11,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160531827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:11,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:11,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:11,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160531827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:11,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:11,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:11,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:11,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:11,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T03:41:11,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-21T03:41:11,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:11,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:11,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:11,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:11,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:11,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:12,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:12,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160532037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:12,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160532037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:12,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:12,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:12,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:12,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:12,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:12,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:12,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-21T03:41:12,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:12,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:12,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:12,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:12,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:12,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:12,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T03:41:12,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/c1bbfd6134ee4838bef41341c8e6166e 2024-11-21T03:41:12,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/f070e7b2248e46fca3aedaa60fe0ad65 is 50, key is test_row_0/C:col10/1732160471365/Put/seqid=0 2024-11-21T03:41:12,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742137_1313 (size=12301) 2024-11-21T03:41:12,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/f070e7b2248e46fca3aedaa60fe0ad65 2024-11-21T03:41:12,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/6e22cca4cc2849a9bded90f4568d820b as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/6e22cca4cc2849a9bded90f4568d820b 2024-11-21T03:41:12,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/6e22cca4cc2849a9bded90f4568d820b, entries=200, sequenceid=295, filesize=14.4 K 2024-11-21T03:41:12,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/c1bbfd6134ee4838bef41341c8e6166e as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/c1bbfd6134ee4838bef41341c8e6166e 2024-11-21T03:41:12,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/c1bbfd6134ee4838bef41341c8e6166e, entries=150, sequenceid=295, filesize=12.0 K 2024-11-21T03:41:12,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/f070e7b2248e46fca3aedaa60fe0ad65 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/f070e7b2248e46fca3aedaa60fe0ad65 2024-11-21T03:41:12,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/f070e7b2248e46fca3aedaa60fe0ad65, entries=150, sequenceid=295, filesize=12.0 K 2024-11-21T03:41:12,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 17b90c50c9bd5dda59fffca9821d39a4 in 488ms, sequenceid=295, compaction requested=true 2024-11-21T03:41:12,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:12,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:41:12,178 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:12,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:12,178 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:12,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:41:12,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:12,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:41:12,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:12,181 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:12,181 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/B is initiating minor compaction (all files) 2024-11-21T03:41:12,181 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/B in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:12,181 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/f2b48261f33a4037ba935179ddc105d5, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/2740747bfe1c40a684c1ae6d46d2e78b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/c1bbfd6134ee4838bef41341c8e6166e] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=36.5 K 2024-11-21T03:41:12,181 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39807 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:12,181 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/A is initiating minor compaction (all files) 2024-11-21T03:41:12,181 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/A in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:12,182 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/507e14ee630849bcbf69e59653ec1db1, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/f7731ee868f94e20be88468c7a21881b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/6e22cca4cc2849a9bded90f4568d820b] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=38.9 K 2024-11-21T03:41:12,182 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting f2b48261f33a4037ba935179ddc105d5, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732160470340 2024-11-21T03:41:12,183 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 507e14ee630849bcbf69e59653ec1db1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732160470340 2024-11-21T03:41:12,183 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 2740747bfe1c40a684c1ae6d46d2e78b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732160470687 2024-11-21T03:41:12,184 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7731ee868f94e20be88468c7a21881b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732160470687 2024-11-21T03:41:12,184 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting c1bbfd6134ee4838bef41341c8e6166e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732160471365 2024-11-21T03:41:12,184 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e22cca4cc2849a9bded90f4568d820b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732160471360 2024-11-21T03:41:12,193 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#B#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:12,193 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/cbbc65f18a9c4e878f6cd1b1b22a8235 is 50, key is test_row_0/B:col10/1732160471365/Put/seqid=0 2024-11-21T03:41:12,198 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#A#compaction#268 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:12,198 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/7699ba77261b4d6281410d12dc8c7fba is 50, key is test_row_0/A:col10/1732160471365/Put/seqid=0 2024-11-21T03:41:12,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742138_1314 (size=13017) 2024-11-21T03:41:12,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-21T03:41:12,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:12,220 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 17b90c50c9bd5dda59fffca9821d39a4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T03:41:12,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/bc52e8d040a546c19b77c697c827a8f3 is 50, key is test_row_0/A:col10/1732160471711/Put/seqid=0 2024-11-21T03:41:12,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742139_1315 (size=13017) 2024-11-21T03:41:12,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742140_1316 (size=12301) 2024-11-21T03:41:12,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:41:12,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T03:41:12,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160532434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:12,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:12,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:12,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:12,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160532440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:12,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:12,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:12,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:12,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160532547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:12,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:12,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:12,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:12,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160532547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:12,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:12,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:12,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:12,617 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/cbbc65f18a9c4e878f6cd1b1b22a8235 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/cbbc65f18a9c4e878f6cd1b1b22a8235 2024-11-21T03:41:12,623 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/B of 17b90c50c9bd5dda59fffca9821d39a4 into cbbc65f18a9c4e878f6cd1b1b22a8235(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:12,623 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:12,623 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/B, priority=13, startTime=1732160472178; duration=0sec 2024-11-21T03:41:12,623 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:12,623 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:B 2024-11-21T03:41:12,623 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:12,625 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:12,625 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/C is initiating minor compaction (all files) 2024-11-21T03:41:12,625 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/C in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:12,625 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/c51c4282209445bbbec0b7ce132f4e8f, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/9ef45713d45a4b788149545c2d0dfed9, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/f070e7b2248e46fca3aedaa60fe0ad65] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=36.5 K 2024-11-21T03:41:12,625 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting c51c4282209445bbbec0b7ce132f4e8f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732160470340 2024-11-21T03:41:12,626 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ef45713d45a4b788149545c2d0dfed9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732160470687 2024-11-21T03:41:12,626 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting f070e7b2248e46fca3aedaa60fe0ad65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732160471365 2024-11-21T03:41:12,634 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#C#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:12,634 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/06a38cd506d743368527099181b3447e is 50, key is test_row_0/C:col10/1732160471365/Put/seqid=0 2024-11-21T03:41:12,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742141_1317 (size=13017) 2024-11-21T03:41:12,645 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/06a38cd506d743368527099181b3447e as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/06a38cd506d743368527099181b3447e 2024-11-21T03:41:12,652 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/C of 17b90c50c9bd5dda59fffca9821d39a4 into 06a38cd506d743368527099181b3447e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:12,652 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:12,652 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/C, priority=13, startTime=1732160472178; duration=0sec 2024-11-21T03:41:12,652 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:12,652 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:C 2024-11-21T03:41:12,652 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/7699ba77261b4d6281410d12dc8c7fba as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/7699ba77261b4d6281410d12dc8c7fba 2024-11-21T03:41:12,658 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/A of 17b90c50c9bd5dda59fffca9821d39a4 into 7699ba77261b4d6281410d12dc8c7fba(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:12,658 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:12,658 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/A, priority=13, startTime=1732160472178; duration=0sec 2024-11-21T03:41:12,658 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:12,658 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:A 2024-11-21T03:41:12,673 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/bc52e8d040a546c19b77c697c827a8f3 2024-11-21T03:41:12,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/47d9beef858d455d87833a08e8bd9177 is 50, key is test_row_0/B:col10/1732160471711/Put/seqid=0 2024-11-21T03:41:12,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742142_1318 (size=12301) 2024-11-21T03:41:12,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:12,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160532757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:12,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:12,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160532757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:12,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:12,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:12,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:12,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:12,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:12,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:12,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T03:41:13,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:13,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160533067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:13,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:13,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:13,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160533067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:13,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:13,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:13,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:13,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:13,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:13,085 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/47d9beef858d455d87833a08e8bd9177 2024-11-21T03:41:13,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/0ef6a856d7a04b7d94eda1ceb5aff7d7 is 50, key is test_row_0/C:col10/1732160471711/Put/seqid=0 2024-11-21T03:41:13,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742143_1319 (size=12301) 2024-11-21T03:41:13,503 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/0ef6a856d7a04b7d94eda1ceb5aff7d7 2024-11-21T03:41:13,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/bc52e8d040a546c19b77c697c827a8f3 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/bc52e8d040a546c19b77c697c827a8f3 2024-11-21T03:41:13,511 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/bc52e8d040a546c19b77c697c827a8f3, entries=150, sequenceid=307, filesize=12.0 K 2024-11-21T03:41:13,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/47d9beef858d455d87833a08e8bd9177 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/47d9beef858d455d87833a08e8bd9177 2024-11-21T03:41:13,517 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/47d9beef858d455d87833a08e8bd9177, entries=150, sequenceid=307, filesize=12.0 K 2024-11-21T03:41:13,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/0ef6a856d7a04b7d94eda1ceb5aff7d7 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/0ef6a856d7a04b7d94eda1ceb5aff7d7 2024-11-21T03:41:13,522 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/0ef6a856d7a04b7d94eda1ceb5aff7d7, entries=150, sequenceid=307, filesize=12.0 K 2024-11-21T03:41:13,523 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 17b90c50c9bd5dda59fffca9821d39a4 in 1303ms, sequenceid=307, compaction requested=false 2024-11-21T03:41:13,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:13,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:13,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-21T03:41:13,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-21T03:41:13,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-21T03:41:13,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7640 sec 2024-11-21T03:41:13,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.7690 sec 2024-11-21T03:41:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:41:13,578 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:41:13,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17b90c50c9bd5dda59fffca9821d39a4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T03:41:13,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/a0a8fc0441fe4cfbafe07b47739d2fd3 is 50, key is test_row_0/A:col10/1732160473578/Put/seqid=0 2024-11-21T03:41:13,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742144_1320 (size=12301) 2024-11-21T03:41:13,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:13,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160533591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:13,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:13,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:13,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:13,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/a0a8fc0441fe4cfbafe07b47739d2fd3 2024-11-21T03:41:13,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:13,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160533594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:13,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/bb1c3213f28c45debab044b3562fbfff is 50, key is test_row_0/B:col10/1732160473578/Put/seqid=0 2024-11-21T03:41:13,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742145_1321 (size=12301) 2024-11-21T03:41:13,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/bb1c3213f28c45debab044b3562fbfff 2024-11-21T03:41:13,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/871aee8449554f29914f45a3f859f0aa is 50, key is test_row_0/C:col10/1732160473578/Put/seqid=0 2024-11-21T03:41:13,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742146_1322 (size=12301) 2024-11-21T03:41:13,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:13,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160533696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:13,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:13,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:13,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:13,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:13,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160533707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:13,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:13,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T03:41:13,907 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:41:13,908 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:41:13,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-21T03:41:13,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-21T03:41:13,909 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:41:13,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:13,910 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:41:13,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160533907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:13,910 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:41:13,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:13,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:13,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:13,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:13,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160533916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:13,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:14,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-21T03:41:14,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/871aee8449554f29914f45a3f859f0aa 2024-11-21T03:41:14,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/a0a8fc0441fe4cfbafe07b47739d2fd3 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/a0a8fc0441fe4cfbafe07b47739d2fd3 2024-11-21T03:41:14,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/a0a8fc0441fe4cfbafe07b47739d2fd3, entries=150, sequenceid=336, filesize=12.0 K 2024-11-21T03:41:14,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/bb1c3213f28c45debab044b3562fbfff as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/bb1c3213f28c45debab044b3562fbfff 2024-11-21T03:41:14,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/bb1c3213f28c45debab044b3562fbfff, entries=150, sequenceid=336, filesize=12.0 K 2024-11-21T03:41:14,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/871aee8449554f29914f45a3f859f0aa as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/871aee8449554f29914f45a3f859f0aa 2024-11-21T03:41:14,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/871aee8449554f29914f45a3f859f0aa, entries=150, sequenceid=336, filesize=12.0 K 2024-11-21T03:41:14,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 17b90c50c9bd5dda59fffca9821d39a4 in 473ms, sequenceid=336, compaction requested=true 2024-11-21T03:41:14,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:14,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:41:14,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:14,051 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:14,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:41:14,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:14,051 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:14,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:41:14,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:14,052 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:14,052 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:14,052 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/B is initiating minor compaction (all files) 2024-11-21T03:41:14,052 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/A is initiating minor compaction (all files) 2024-11-21T03:41:14,052 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/B in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:14,052 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/A in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:14,052 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/7699ba77261b4d6281410d12dc8c7fba, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/bc52e8d040a546c19b77c697c827a8f3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/a0a8fc0441fe4cfbafe07b47739d2fd3] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=36.7 K 2024-11-21T03:41:14,052 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/cbbc65f18a9c4e878f6cd1b1b22a8235, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/47d9beef858d455d87833a08e8bd9177, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/bb1c3213f28c45debab044b3562fbfff] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=36.7 K 2024-11-21T03:41:14,053 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting cbbc65f18a9c4e878f6cd1b1b22a8235, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732160471365 2024-11-21T03:41:14,053 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7699ba77261b4d6281410d12dc8c7fba, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732160471365 2024-11-21T03:41:14,053 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 47d9beef858d455d87833a08e8bd9177, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732160471699 2024-11-21T03:41:14,053 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc52e8d040a546c19b77c697c827a8f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732160471699 2024-11-21T03:41:14,053 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting bb1c3213f28c45debab044b3562fbfff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732160472369 2024-11-21T03:41:14,053 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0a8fc0441fe4cfbafe07b47739d2fd3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732160472369 2024-11-21T03:41:14,060 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#B#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:14,060 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/dbfd34aef80545efadcb5d1d5bd39b9e is 50, key is test_row_0/B:col10/1732160473578/Put/seqid=0 2024-11-21T03:41:14,062 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#A#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:14,063 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/cfb0a5a6b47f4325a6d2f91618aada03 is 50, key is test_row_0/A:col10/1732160473578/Put/seqid=0 2024-11-21T03:41:14,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-21T03:41:14,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:14,064 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing 17b90c50c9bd5dda59fffca9821d39a4 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-21T03:41:14,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742147_1323 (size=13119) 2024-11-21T03:41:14,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/6e7c03da55e347c3af71032c33991ecb is 50, key is test_row_0/A:col10/1732160473593/Put/seqid=0 2024-11-21T03:41:14,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742148_1324 (size=13119) 2024-11-21T03:41:14,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742149_1325 (size=12301) 2024-11-21T03:41:14,074 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/6e7c03da55e347c3af71032c33991ecb 2024-11-21T03:41:14,075 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/dbfd34aef80545efadcb5d1d5bd39b9e as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/dbfd34aef80545efadcb5d1d5bd39b9e 2024-11-21T03:41:14,084 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/B of 17b90c50c9bd5dda59fffca9821d39a4 into dbfd34aef80545efadcb5d1d5bd39b9e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:14,084 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:14,084 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/B, priority=13, startTime=1732160474051; duration=0sec 2024-11-21T03:41:14,084 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:14,084 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:B 2024-11-21T03:41:14,084 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:14,086 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:14,086 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/C is initiating minor compaction (all files) 2024-11-21T03:41:14,086 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/C in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:14,086 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/06a38cd506d743368527099181b3447e, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/0ef6a856d7a04b7d94eda1ceb5aff7d7, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/871aee8449554f29914f45a3f859f0aa] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=36.7 K 2024-11-21T03:41:14,086 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 06a38cd506d743368527099181b3447e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732160471365 2024-11-21T03:41:14,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/b4b34c62c3d94cbeb63a239290088f3c is 50, key is test_row_0/B:col10/1732160473593/Put/seqid=0 2024-11-21T03:41:14,089 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ef6a856d7a04b7d94eda1ceb5aff7d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1732160471699 2024-11-21T03:41:14,089 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 871aee8449554f29914f45a3f859f0aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732160472369 2024-11-21T03:41:14,098 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#C#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:14,099 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/81de1ad7a9eb4953ba28f4db245527f2 is 50, key is test_row_0/C:col10/1732160473578/Put/seqid=0 2024-11-21T03:41:14,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742150_1326 (size=12301) 2024-11-21T03:41:14,117 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/b4b34c62c3d94cbeb63a239290088f3c 2024-11-21T03:41:14,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742151_1327 (size=13119) 2024-11-21T03:41:14,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/4fdb99acae984466bc4e8d89af796876 is 50, key is test_row_0/C:col10/1732160473593/Put/seqid=0 2024-11-21T03:41:14,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742152_1328 (size=12301) 2024-11-21T03:41:14,164 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/4fdb99acae984466bc4e8d89af796876 2024-11-21T03:41:14,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/6e7c03da55e347c3af71032c33991ecb as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/6e7c03da55e347c3af71032c33991ecb 2024-11-21T03:41:14,173 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/6e7c03da55e347c3af71032c33991ecb, entries=150, sequenceid=346, filesize=12.0 K 2024-11-21T03:41:14,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/b4b34c62c3d94cbeb63a239290088f3c as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/b4b34c62c3d94cbeb63a239290088f3c 2024-11-21T03:41:14,178 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/b4b34c62c3d94cbeb63a239290088f3c, entries=150, sequenceid=346, filesize=12.0 K 2024-11-21T03:41:14,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/4fdb99acae984466bc4e8d89af796876 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/4fdb99acae984466bc4e8d89af796876 2024-11-21T03:41:14,183 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/4fdb99acae984466bc4e8d89af796876, entries=150, sequenceid=346, filesize=12.0 K 2024-11-21T03:41:14,184 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for 17b90c50c9bd5dda59fffca9821d39a4 in 120ms, sequenceid=346, compaction requested=false 2024-11-21T03:41:14,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:14,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:14,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-21T03:41:14,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-21T03:41:14,188 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-21T03:41:14,188 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 276 msec 2024-11-21T03:41:14,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 281 msec 2024-11-21T03:41:14,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-21T03:41:14,227 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:41:14,227 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:41:14,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees 2024-11-21T03:41:14,229 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:41:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-21T03:41:14,229 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=88, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:41:14,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:41:14,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:41:14,231 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:41:14,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17b90c50c9bd5dda59fffca9821d39a4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:41:14,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/19cdb328822449ae9f56d794868dc293 is 50, key is test_row_0/A:col10/1732160474230/Put/seqid=0 2024-11-21T03:41:14,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742153_1329 (size=14741) 2024-11-21T03:41:14,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:14,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160534304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:14,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:14,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:14,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:14,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:14,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160534305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:14,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:14,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:14,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:14,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-21T03:41:14,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-21T03:41:14,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:14,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:14,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:14,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:14,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:14,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:14,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160534417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:14,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:14,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:14,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:14,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160534417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:14,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:14,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:14,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:14,473 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/cfb0a5a6b47f4325a6d2f91618aada03 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/cfb0a5a6b47f4325a6d2f91618aada03 2024-11-21T03:41:14,482 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/A of 17b90c50c9bd5dda59fffca9821d39a4 into cfb0a5a6b47f4325a6d2f91618aada03(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:14,482 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:14,482 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/A, priority=13, startTime=1732160474051; duration=0sec 2024-11-21T03:41:14,482 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:14,482 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:A 2024-11-21T03:41:14,533 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/81de1ad7a9eb4953ba28f4db245527f2 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/81de1ad7a9eb4953ba28f4db245527f2 2024-11-21T03:41:14,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-21T03:41:14,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:14,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:14,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:14,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:14,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:14,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:14,539 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/C of 17b90c50c9bd5dda59fffca9821d39a4 into 81de1ad7a9eb4953ba28f4db245527f2(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:14,539 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:14,539 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/C, priority=13, startTime=1732160474051; duration=0sec 2024-11-21T03:41:14,540 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:14,540 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:C 2024-11-21T03:41:14,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-21T03:41:14,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160534627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:14,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:14,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:14,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:14,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:14,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160534627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:14,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:14,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:14,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:14,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/19cdb328822449ae9f56d794868dc293 2024-11-21T03:41:14,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/93b35942b1ec4a8e9aaaeda8f2871b71 is 50, key is test_row_0/B:col10/1732160474230/Put/seqid=0 2024-11-21T03:41:14,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742154_1330 (size=12301) 2024-11-21T03:41:14,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/93b35942b1ec4a8e9aaaeda8f2871b71 2024-11-21T03:41:14,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/1ce084bde8a841ea8727315ebf4ff28f is 50, key is test_row_0/C:col10/1732160474230/Put/seqid=0 2024-11-21T03:41:14,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742155_1331 (size=12301) 2024-11-21T03:41:14,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-21T03:41:14,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:14,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:14,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:14,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:14,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:14,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:14,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-21T03:41:14,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:14,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:14,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:14,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:14,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:14,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:14,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-21T03:41:14,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160534936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:14,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160534937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:14,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:14,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:14,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:14,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-21T03:41:14,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:14,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:14,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:14,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] handler.RSProcedureHandler(58): pid=89 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:14,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=89 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:14,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=89 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:15,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/1ce084bde8a841ea8727315ebf4ff28f 2024-11-21T03:41:15,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/19cdb328822449ae9f56d794868dc293 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/19cdb328822449ae9f56d794868dc293 2024-11-21T03:41:15,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/19cdb328822449ae9f56d794868dc293, entries=200, sequenceid=358, filesize=14.4 K 2024-11-21T03:41:15,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/93b35942b1ec4a8e9aaaeda8f2871b71 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/93b35942b1ec4a8e9aaaeda8f2871b71 2024-11-21T03:41:15,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/93b35942b1ec4a8e9aaaeda8f2871b71, entries=150, sequenceid=358, filesize=12.0 K 2024-11-21T03:41:15,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/1ce084bde8a841ea8727315ebf4ff28f as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/1ce084bde8a841ea8727315ebf4ff28f 2024-11-21T03:41:15,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/1ce084bde8a841ea8727315ebf4ff28f, entries=150, sequenceid=358, filesize=12.0 K 2024-11-21T03:41:15,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 17b90c50c9bd5dda59fffca9821d39a4 in 846ms, sequenceid=358, compaction requested=true 2024-11-21T03:41:15,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:15,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:41:15,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:15,077 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:15,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:41:15,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:15,077 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:15,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:41:15,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:15,078 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:15,078 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:15,078 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/A is initiating minor compaction (all files) 2024-11-21T03:41:15,078 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/B is initiating minor compaction (all files) 2024-11-21T03:41:15,079 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/B in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:15,079 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/A in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:15,079 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/cfb0a5a6b47f4325a6d2f91618aada03, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/6e7c03da55e347c3af71032c33991ecb, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/19cdb328822449ae9f56d794868dc293] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=39.2 K 2024-11-21T03:41:15,079 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/dbfd34aef80545efadcb5d1d5bd39b9e, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/b4b34c62c3d94cbeb63a239290088f3c, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/93b35942b1ec4a8e9aaaeda8f2871b71] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=36.8 K 2024-11-21T03:41:15,079 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfb0a5a6b47f4325a6d2f91618aada03, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732160472369 2024-11-21T03:41:15,079 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting dbfd34aef80545efadcb5d1d5bd39b9e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732160472369 2024-11-21T03:41:15,079 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e7c03da55e347c3af71032c33991ecb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732160473586 2024-11-21T03:41:15,079 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting b4b34c62c3d94cbeb63a239290088f3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732160473586 2024-11-21T03:41:15,079 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19cdb328822449ae9f56d794868dc293, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732160474225 2024-11-21T03:41:15,080 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 93b35942b1ec4a8e9aaaeda8f2871b71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732160474227 2024-11-21T03:41:15,087 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#B#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:15,087 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#A#compaction#286 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:15,087 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/0d71a05f59e14364b990da73795d4315 is 50, key is test_row_0/B:col10/1732160474230/Put/seqid=0 2024-11-21T03:41:15,088 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/56130b51ca4247d498af97542083fdfc is 50, key is test_row_0/A:col10/1732160474230/Put/seqid=0 2024-11-21T03:41:15,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742157_1333 (size=13221) 2024-11-21T03:41:15,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742156_1332 (size=13221) 2024-11-21T03:41:15,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=89 2024-11-21T03:41:15,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:15,145 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2902): Flushing 17b90c50c9bd5dda59fffca9821d39a4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T03:41:15,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/91ebe45eed974291ac013929a8739ac9 is 50, key is test_row_0/A:col10/1732160474296/Put/seqid=0 2024-11-21T03:41:15,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742158_1334 (size=12301) 2024-11-21T03:41:15,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-21T03:41:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:41:15,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:15,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:15,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160535460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:15,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:15,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:15,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160535461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:15,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:15,464 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:15,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:15,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:15,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:15,500 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/0d71a05f59e14364b990da73795d4315 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/0d71a05f59e14364b990da73795d4315 2024-11-21T03:41:15,503 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/B of 17b90c50c9bd5dda59fffca9821d39a4 into 0d71a05f59e14364b990da73795d4315(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:15,503 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:15,503 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/B, priority=13, startTime=1732160475077; duration=0sec 2024-11-21T03:41:15,504 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:15,504 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:B 2024-11-21T03:41:15,504 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:15,504 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:15,504 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/C is initiating minor compaction (all files) 2024-11-21T03:41:15,504 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/C in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:15,505 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/81de1ad7a9eb4953ba28f4db245527f2, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/4fdb99acae984466bc4e8d89af796876, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/1ce084bde8a841ea8727315ebf4ff28f] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=36.8 K 2024-11-21T03:41:15,505 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 81de1ad7a9eb4953ba28f4db245527f2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732160472369 2024-11-21T03:41:15,505 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fdb99acae984466bc4e8d89af796876, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1732160473586 2024-11-21T03:41:15,505 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ce084bde8a841ea8727315ebf4ff28f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732160474227 2024-11-21T03:41:15,510 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/56130b51ca4247d498af97542083fdfc as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/56130b51ca4247d498af97542083fdfc 2024-11-21T03:41:15,513 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#C#compaction#288 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:15,514 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/9b6fe42d98ee4fd694908188f9521239 is 50, key is test_row_0/C:col10/1732160474230/Put/seqid=0 2024-11-21T03:41:15,514 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/A of 17b90c50c9bd5dda59fffca9821d39a4 into 56130b51ca4247d498af97542083fdfc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:15,515 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:15,515 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/A, priority=13, startTime=1732160475077; duration=0sec 2024-11-21T03:41:15,515 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:15,515 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:A 2024-11-21T03:41:15,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742159_1335 (size=13221) 2024-11-21T03:41:15,529 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/9b6fe42d98ee4fd694908188f9521239 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/9b6fe42d98ee4fd694908188f9521239 2024-11-21T03:41:15,532 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/C of 17b90c50c9bd5dda59fffca9821d39a4 into 9b6fe42d98ee4fd694908188f9521239(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:15,533 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:15,533 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/C, priority=13, startTime=1732160475077; duration=0sec 2024-11-21T03:41:15,533 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:15,533 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:C 2024-11-21T03:41:15,554 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/91ebe45eed974291ac013929a8739ac9 2024-11-21T03:41:15,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/699e0033110541bb9d2195544291e011 is 50, key is test_row_0/B:col10/1732160474296/Put/seqid=0 2024-11-21T03:41:15,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742160_1336 (size=12301) 2024-11-21T03:41:15,563 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/699e0033110541bb9d2195544291e011 2024-11-21T03:41:15,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160535566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:15,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160535567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:15,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/764166e092474e50af35a5edfc831c2d is 50, key is test_row_0/C:col10/1732160474296/Put/seqid=0 2024-11-21T03:41:15,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742161_1337 (size=12301) 2024-11-21T03:41:15,573 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/764166e092474e50af35a5edfc831c2d 2024-11-21T03:41:15,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/91ebe45eed974291ac013929a8739ac9 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/91ebe45eed974291ac013929a8739ac9 2024-11-21T03:41:15,582 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/91ebe45eed974291ac013929a8739ac9, entries=150, sequenceid=385, filesize=12.0 K 2024-11-21T03:41:15,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/699e0033110541bb9d2195544291e011 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/699e0033110541bb9d2195544291e011 2024-11-21T03:41:15,585 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/699e0033110541bb9d2195544291e011, entries=150, sequenceid=385, filesize=12.0 K 2024-11-21T03:41:15,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/764166e092474e50af35a5edfc831c2d as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/764166e092474e50af35a5edfc831c2d 2024-11-21T03:41:15,591 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/764166e092474e50af35a5edfc831c2d, entries=150, sequenceid=385, filesize=12.0 K 2024-11-21T03:41:15,591 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 17b90c50c9bd5dda59fffca9821d39a4 in 446ms, sequenceid=385, compaction requested=false 2024-11-21T03:41:15,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.HRegion(2603): Flush status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:15,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:15,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=89}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=89 2024-11-21T03:41:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=89 2024-11-21T03:41:15,594 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-21T03:41:15,594 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3630 sec 2024-11-21T03:41:15,596 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=88, table=TestAcidGuarantees in 1.3680 sec 2024-11-21T03:41:15,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:41:15,778 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:41:15,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17b90c50c9bd5dda59fffca9821d39a4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T03:41:15,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/79ef25fd007c419baf9d5e8c387a0212 is 50, key is test_row_0/A:col10/1732160475777/Put/seqid=0 2024-11-21T03:41:15,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742162_1338 (size=14741) 2024-11-21T03:41:15,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:15,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160535820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:15,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:15,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:15,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:15,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:15,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160535821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:15,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:15,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:15,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:15,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:15,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160535926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:15,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:15,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:15,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:15,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160535937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:16,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:16,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160536137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:16,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:16,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:16,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:16,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:16,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160536146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:16,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:16,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:16,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:16,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/79ef25fd007c419baf9d5e8c387a0212 2024-11-21T03:41:16,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/41c6accb2e204f6f9e33b6a84f2fd103 is 50, key is test_row_0/B:col10/1732160475777/Put/seqid=0 2024-11-21T03:41:16,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742163_1339 (size=12301) 2024-11-21T03:41:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-21T03:41:16,377 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:41:16,377 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:41:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees 2024-11-21T03:41:16,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-21T03:41:16,379 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:41:16,380 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=90, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:41:16,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:41:16,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:16,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160536446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:16,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:16,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:16,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:16,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:16,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160536457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:16,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:16,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:16,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-21T03:41:16,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-21T03:41:16,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:16,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:16,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:16,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] handler.RSProcedureHandler(58): pid=91 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:16,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=91 java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:16,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=91 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:16,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/41c6accb2e204f6f9e33b6a84f2fd103 2024-11-21T03:41:16,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/9e54796d0b8f4dd5aa80aa3bfb39429a is 50, key is test_row_0/C:col10/1732160475777/Put/seqid=0 2024-11-21T03:41:16,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742164_1340 (size=12301) 2024-11-21T03:41:16,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/9e54796d0b8f4dd5aa80aa3bfb39429a 2024-11-21T03:41:16,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/79ef25fd007c419baf9d5e8c387a0212 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/79ef25fd007c419baf9d5e8c387a0212 2024-11-21T03:41:16,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/79ef25fd007c419baf9d5e8c387a0212, entries=200, sequenceid=400, filesize=14.4 K 2024-11-21T03:41:16,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/41c6accb2e204f6f9e33b6a84f2fd103 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/41c6accb2e204f6f9e33b6a84f2fd103 2024-11-21T03:41:16,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/41c6accb2e204f6f9e33b6a84f2fd103, entries=150, sequenceid=400, filesize=12.0 K 2024-11-21T03:41:16,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/9e54796d0b8f4dd5aa80aa3bfb39429a as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/9e54796d0b8f4dd5aa80aa3bfb39429a 2024-11-21T03:41:16,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/9e54796d0b8f4dd5aa80aa3bfb39429a, entries=150, sequenceid=400, filesize=12.0 K 2024-11-21T03:41:16,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 17b90c50c9bd5dda59fffca9821d39a4 in 874ms, sequenceid=400, compaction requested=true 2024-11-21T03:41:16,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:16,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:41:16,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:16,652 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:16,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:41:16,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:16,652 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:16,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:41:16,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:16,655 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:16,655 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/B is initiating minor compaction (all files) 2024-11-21T03:41:16,655 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:16,655 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/B in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:16,656 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/A is initiating minor compaction (all files) 2024-11-21T03:41:16,656 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/A in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:16,656 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/0d71a05f59e14364b990da73795d4315, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/699e0033110541bb9d2195544291e011, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/41c6accb2e204f6f9e33b6a84f2fd103] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=36.9 K 2024-11-21T03:41:16,656 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/56130b51ca4247d498af97542083fdfc, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/91ebe45eed974291ac013929a8739ac9, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/79ef25fd007c419baf9d5e8c387a0212] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=39.3 K 2024-11-21T03:41:16,660 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56130b51ca4247d498af97542083fdfc, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732160474227 2024-11-21T03:41:16,660 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d71a05f59e14364b990da73795d4315, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732160474227 2024-11-21T03:41:16,660 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91ebe45eed974291ac013929a8739ac9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732160474296 2024-11-21T03:41:16,660 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 699e0033110541bb9d2195544291e011, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732160474296 2024-11-21T03:41:16,660 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79ef25fd007c419baf9d5e8c387a0212, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732160475456 2024-11-21T03:41:16,660 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 41c6accb2e204f6f9e33b6a84f2fd103, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732160475459 2024-11-21T03:41:16,672 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#B#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:16,673 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/7f65f6bb4ee04d06a1cc518dcdd1854d is 50, key is test_row_0/B:col10/1732160475777/Put/seqid=0 2024-11-21T03:41:16,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=91 2024-11-21T03:41:16,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:16,684 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2902): Flushing 17b90c50c9bd5dda59fffca9821d39a4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T03:41:16,695 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#A#compaction#295 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:16,696 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/38cc71c331ec451eb34d3ee55daadaf4 is 50, key is test_row_0/A:col10/1732160475777/Put/seqid=0 2024-11-21T03:41:16,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-21T03:41:16,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/084afc81711d46219e3a66a8327c281d is 50, key is test_row_0/A:col10/1732160475820/Put/seqid=0 2024-11-21T03:41:16,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742166_1342 (size=13323) 2024-11-21T03:41:16,736 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/38cc71c331ec451eb34d3ee55daadaf4 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/38cc71c331ec451eb34d3ee55daadaf4 2024-11-21T03:41:16,743 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/A of 17b90c50c9bd5dda59fffca9821d39a4 into 38cc71c331ec451eb34d3ee55daadaf4(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:16,743 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:16,743 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/A, priority=13, startTime=1732160476652; duration=0sec 2024-11-21T03:41:16,743 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:16,745 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:A 2024-11-21T03:41:16,746 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:16,746 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:16,747 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/C is initiating minor compaction (all files) 2024-11-21T03:41:16,747 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/C in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:16,747 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/9b6fe42d98ee4fd694908188f9521239, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/764166e092474e50af35a5edfc831c2d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/9e54796d0b8f4dd5aa80aa3bfb39429a] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=36.9 K 2024-11-21T03:41:16,747 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b6fe42d98ee4fd694908188f9521239, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1732160474227 2024-11-21T03:41:16,747 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 764166e092474e50af35a5edfc831c2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732160474296 2024-11-21T03:41:16,747 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e54796d0b8f4dd5aa80aa3bfb39429a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732160475459 2024-11-21T03:41:16,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742167_1343 (size=12301) 2024-11-21T03:41:16,755 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#C#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:16,755 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/466116c279494b429efde4837d361bdc is 50, key is test_row_0/C:col10/1732160475777/Put/seqid=0 2024-11-21T03:41:16,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742165_1341 (size=13323) 2024-11-21T03:41:16,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742168_1344 (size=13323) 2024-11-21T03:41:16,776 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/7f65f6bb4ee04d06a1cc518dcdd1854d as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/7f65f6bb4ee04d06a1cc518dcdd1854d 2024-11-21T03:41:16,779 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/466116c279494b429efde4837d361bdc as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/466116c279494b429efde4837d361bdc 2024-11-21T03:41:16,787 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/B of 17b90c50c9bd5dda59fffca9821d39a4 into 7f65f6bb4ee04d06a1cc518dcdd1854d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:16,787 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:16,787 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/B, priority=13, startTime=1732160476652; duration=0sec 2024-11-21T03:41:16,787 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:16,787 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:B 2024-11-21T03:41:16,789 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/C of 17b90c50c9bd5dda59fffca9821d39a4 into 466116c279494b429efde4837d361bdc(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:16,789 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:16,789 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/C, priority=13, startTime=1732160476652; duration=0sec 2024-11-21T03:41:16,789 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:16,789 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:C 2024-11-21T03:41:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:41:16,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. as already flushing 2024-11-21T03:41:16,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:16,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160536982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:16,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:16,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:16,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:16,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:16,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160536989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:16,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:16,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:16,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:17,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-21T03:41:17,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160537096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:17,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:17,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:17,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:17,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160537096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:17,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:17,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:17,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:17,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50890 deadline: 1732160537147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:17,148 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_2' in 17b90c50c9bd5dda59fffca9821d39a4 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:41:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:17,153 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/084afc81711d46219e3a66a8327c281d 2024-11-21T03:41:17,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:17,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50918 deadline: 1732160537167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:17,168 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_0' in 17b90c50c9bd5dda59fffca9821d39a4 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:41:17,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:17,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:17,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:17,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/c5eb5ae95caf41cf88f10a64b197b94b is 50, key is test_row_0/B:col10/1732160475820/Put/seqid=0 2024-11-21T03:41:17,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:17,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50974 deadline: 1732160537186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:17,188 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_1' in 17b90c50c9bd5dda59fffca9821d39a4 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:41:17,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:17,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:17,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:17,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742169_1345 (size=12301) 2024-11-21T03:41:17,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:17,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160537307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:17,307 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:17,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:17,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:17,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:17,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160537307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:17,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:17,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:17,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:17,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-21T03:41:17,593 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/c5eb5ae95caf41cf88f10a64b197b94b 2024-11-21T03:41:17,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/6b6be39e6994483abf8ab51dec23f971 is 50, key is test_row_0/C:col10/1732160475820/Put/seqid=0 2024-11-21T03:41:17,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:17,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50950 deadline: 1732160537617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:17,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:17,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:50960 deadline: 1732160537617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:41:17,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742170_1346 (size=12301) 2024-11-21T03:41:17,622 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/6b6be39e6994483abf8ab51dec23f971 2024-11-21T03:41:17,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/084afc81711d46219e3a66a8327c281d as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/084afc81711d46219e3a66a8327c281d 2024-11-21T03:41:17,635 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/084afc81711d46219e3a66a8327c281d, entries=150, sequenceid=424, filesize=12.0 K 2024-11-21T03:41:17,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/c5eb5ae95caf41cf88f10a64b197b94b as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/c5eb5ae95caf41cf88f10a64b197b94b 2024-11-21T03:41:17,642 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/c5eb5ae95caf41cf88f10a64b197b94b, entries=150, sequenceid=424, filesize=12.0 K 2024-11-21T03:41:17,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/6b6be39e6994483abf8ab51dec23f971 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/6b6be39e6994483abf8ab51dec23f971 2024-11-21T03:41:17,648 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/6b6be39e6994483abf8ab51dec23f971, entries=150, sequenceid=424, filesize=12.0 K 2024-11-21T03:41:17,649 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 17b90c50c9bd5dda59fffca9821d39a4 in 964ms, sequenceid=424, compaction requested=false 2024-11-21T03:41:17,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.HRegion(2603): Flush status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:17,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:17,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=91}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=91 2024-11-21T03:41:17,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=91 2024-11-21T03:41:17,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-21T03:41:17,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2710 sec 2024-11-21T03:41:17,658 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=90, table=TestAcidGuarantees in 1.2770 sec 2024-11-21T03:41:17,720 INFO [Thread-1174 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1174. 2024-11-21T03:41:17,720 DEBUG [Thread-1174 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:41:17,720 DEBUG [Thread-1174 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:17,720 DEBUG [Thread-1174 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:17,720 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:41:17,721 INFO [Thread-1168 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1168. 2024-11-21T03:41:17,721 DEBUG [Thread-1168 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:41:17,721 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:17,721 INFO [Thread-1172 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1172. 2024-11-21T03:41:17,721 DEBUG [Thread-1172 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:41:17,721 DEBUG [Thread-1172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:17,721 DEBUG [Thread-1168 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:17,721 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:41:17,721 DEBUG [Thread-1172 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:17,721 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:41:17,721 INFO [Thread-1176 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1176. 2024-11-21T03:41:17,721 DEBUG [Thread-1176 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:41:17,721 DEBUG [Thread-1176 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:17,721 DEBUG [Thread-1176 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:17,721 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:41:17,723 INFO [Thread-1170 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1170. 2024-11-21T03:41:17,723 DEBUG [Thread-1170 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:41:17,723 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:17,724 DEBUG [Thread-1170 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:17,724 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:41:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:41:18,131 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:41:18,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17b90c50c9bd5dda59fffca9821d39a4 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T03:41:18,131 INFO [Thread-1157 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1157. 2024-11-21T03:41:18,131 INFO [Thread-1165 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1165. 2024-11-21T03:41:18,132 DEBUG [Thread-1157 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:41:18,132 DEBUG [Thread-1165 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:41:18,132 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:18,132 DEBUG [Thread-1165 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:18,132 DEBUG [Thread-1165 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:18,132 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:18,132 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:41:18,132 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:41:18,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/fb075d354c754f45a9958dc9d6b3385a is 50, key is test_row_0/A:col10/1732160478128/Put/seqid=0 2024-11-21T03:41:18,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742171_1347 (size=12301) 2024-11-21T03:41:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-21T03:41:18,527 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:41:18,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/fb075d354c754f45a9958dc9d6b3385a 2024-11-21T03:41:18,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/51cb6c37fc134d4da781d0c579dde579 is 50, key is test_row_0/B:col10/1732160478128/Put/seqid=0 2024-11-21T03:41:18,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742172_1348 (size=12301) 2024-11-21T03:41:18,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/51cb6c37fc134d4da781d0c579dde579 2024-11-21T03:41:18,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/cfab43ca85a741358d2f390638d183dd is 50, key is test_row_0/C:col10/1732160478128/Put/seqid=0 2024-11-21T03:41:18,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742173_1349 (size=12301) 2024-11-21T03:41:18,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/cfab43ca85a741358d2f390638d183dd 2024-11-21T03:41:18,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/fb075d354c754f45a9958dc9d6b3385a as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/fb075d354c754f45a9958dc9d6b3385a 2024-11-21T03:41:18,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/fb075d354c754f45a9958dc9d6b3385a, entries=150, sequenceid=441, filesize=12.0 K 2024-11-21T03:41:18,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/51cb6c37fc134d4da781d0c579dde579 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/51cb6c37fc134d4da781d0c579dde579 2024-11-21T03:41:18,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/51cb6c37fc134d4da781d0c579dde579, entries=150, sequenceid=441, filesize=12.0 K 2024-11-21T03:41:18,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/cfab43ca85a741358d2f390638d183dd as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/cfab43ca85a741358d2f390638d183dd 2024-11-21T03:41:18,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/cfab43ca85a741358d2f390638d183dd, entries=150, sequenceid=441, filesize=12.0 K 2024-11-21T03:41:18,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for 17b90c50c9bd5dda59fffca9821d39a4 in 454ms, sequenceid=441, compaction requested=true 2024-11-21T03:41:18,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:18,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:41:18,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:18,585 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:18,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:41:18,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:18,585 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:18,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17b90c50c9bd5dda59fffca9821d39a4:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:41:18,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:18,586 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:18,586 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:18,586 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/B is initiating minor compaction (all files) 2024-11-21T03:41:18,586 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/A is initiating minor compaction (all files) 2024-11-21T03:41:18,586 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/B in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:18,586 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/A in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:18,586 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/38cc71c331ec451eb34d3ee55daadaf4, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/084afc81711d46219e3a66a8327c281d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/fb075d354c754f45a9958dc9d6b3385a] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=37.0 K 2024-11-21T03:41:18,586 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/7f65f6bb4ee04d06a1cc518dcdd1854d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/c5eb5ae95caf41cf88f10a64b197b94b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/51cb6c37fc134d4da781d0c579dde579] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=37.0 K 2024-11-21T03:41:18,587 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f65f6bb4ee04d06a1cc518dcdd1854d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732160475459 2024-11-21T03:41:18,587 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38cc71c331ec451eb34d3ee55daadaf4, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732160475459 2024-11-21T03:41:18,587 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting c5eb5ae95caf41cf88f10a64b197b94b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1732160475797 2024-11-21T03:41:18,587 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 084afc81711d46219e3a66a8327c281d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1732160475797 2024-11-21T03:41:18,587 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb075d354c754f45a9958dc9d6b3385a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1732160476981 2024-11-21T03:41:18,587 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 51cb6c37fc134d4da781d0c579dde579, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1732160476981 2024-11-21T03:41:18,594 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#A#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:18,595 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/cfd918622c14418787aa2c5899d377ef is 50, key is test_row_0/A:col10/1732160478128/Put/seqid=0 2024-11-21T03:41:18,597 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#B#compaction#304 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:18,597 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/3139c907a8bc471482de9065d8a793d3 is 50, key is test_row_0/B:col10/1732160478128/Put/seqid=0 2024-11-21T03:41:18,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742174_1350 (size=13425) 2024-11-21T03:41:18,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742175_1351 (size=13425) 2024-11-21T03:41:19,005 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/cfd918622c14418787aa2c5899d377ef as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/cfd918622c14418787aa2c5899d377ef 2024-11-21T03:41:19,006 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/3139c907a8bc471482de9065d8a793d3 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/3139c907a8bc471482de9065d8a793d3 2024-11-21T03:41:19,009 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/A of 17b90c50c9bd5dda59fffca9821d39a4 into cfd918622c14418787aa2c5899d377ef(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:19,009 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:19,010 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/A, priority=13, startTime=1732160478585; duration=0sec 2024-11-21T03:41:19,010 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:19,010 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:A 2024-11-21T03:41:19,010 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:19,011 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/B of 17b90c50c9bd5dda59fffca9821d39a4 into 3139c907a8bc471482de9065d8a793d3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:19,011 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:19,011 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:19,011 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/B, priority=13, startTime=1732160478585; duration=0sec 2024-11-21T03:41:19,011 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): 17b90c50c9bd5dda59fffca9821d39a4/C is initiating minor compaction (all files) 2024-11-21T03:41:19,011 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17b90c50c9bd5dda59fffca9821d39a4/C in TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:19,011 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:19,011 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:B 2024-11-21T03:41:19,011 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/466116c279494b429efde4837d361bdc, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/6b6be39e6994483abf8ab51dec23f971, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/cfab43ca85a741358d2f390638d183dd] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp, totalSize=37.0 K 2024-11-21T03:41:19,011 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 466116c279494b429efde4837d361bdc, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732160475459 2024-11-21T03:41:19,012 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b6be39e6994483abf8ab51dec23f971, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1732160475797 2024-11-21T03:41:19,012 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfab43ca85a741358d2f390638d183dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=441, earliestPutTs=1732160476981 2024-11-21T03:41:19,019 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17b90c50c9bd5dda59fffca9821d39a4#C#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:19,020 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/608d5df4fd6643518323001d29bebf95 is 50, key is test_row_0/C:col10/1732160478128/Put/seqid=0 2024-11-21T03:41:19,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742176_1352 (size=13425) 2024-11-21T03:41:19,032 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/608d5df4fd6643518323001d29bebf95 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/608d5df4fd6643518323001d29bebf95 2024-11-21T03:41:19,037 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17b90c50c9bd5dda59fffca9821d39a4/C of 17b90c50c9bd5dda59fffca9821d39a4 into 608d5df4fd6643518323001d29bebf95(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:19,037 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17b90c50c9bd5dda59fffca9821d39a4: 2024-11-21T03:41:19,037 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., storeName=17b90c50c9bd5dda59fffca9821d39a4/C, priority=13, startTime=1732160478585; duration=0sec 2024-11-21T03:41:19,038 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:19,038 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17b90c50c9bd5dda59fffca9821d39a4:C 2024-11-21T03:41:25,192 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T03:41:27,237 INFO [Thread-1161 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1161. 2024-11-21T03:41:27,237 INFO [Thread-1163 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1163. 2024-11-21T03:41:27,237 DEBUG [Thread-1163 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:41:27,237 DEBUG [Thread-1161 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:41:27,237 INFO [Thread-1159 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1159. 2024-11-21T03:41:27,237 DEBUG [Thread-1163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:27,237 DEBUG [Thread-1161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:27,237 DEBUG [Thread-1159 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:41:27,237 DEBUG [Thread-1159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:27,237 DEBUG [Thread-1161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:27,237 DEBUG [Thread-1163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:27,237 DEBUG [Thread-1159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:27,237 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:41:27,237 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:41:27,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T03:41:27,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 152 2024-11-21T03:41:27,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-21T03:41:27,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-21T03:41:27,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-21T03:41:27,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 162 2024-11-21T03:41:27,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T03:41:27,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4003 2024-11-21T03:41:27,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4003 2024-11-21T03:41:27,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3920 2024-11-21T03:41:27,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4001 2024-11-21T03:41:27,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3990 2024-11-21T03:41:27,238 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T03:41:27,238 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T03:41:27,238 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T03:41:27,238 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:27,238 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:27,238 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:41:27,240 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:41:27,241 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T03:41:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T03:41:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-21T03:41:27,244 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732160487244"}]},"ts":"1732160487244"} 2024-11-21T03:41:27,246 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T03:41:27,246 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T03:41:27,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T03:41:27,248 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17b90c50c9bd5dda59fffca9821d39a4, UNASSIGN}] 2024-11-21T03:41:27,249 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17b90c50c9bd5dda59fffca9821d39a4, UNASSIGN 2024-11-21T03:41:27,250 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=17b90c50c9bd5dda59fffca9821d39a4, regionState=CLOSING, regionLocation=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:27,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=17b90c50c9bd5dda59fffca9821d39a4, UNASSIGN because future has completed 2024-11-21T03:41:27,252 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T03:41:27,252 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889}] 2024-11-21T03:41:27,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-21T03:41:27,405 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:41:27,405 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T03:41:27,405 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing 17b90c50c9bd5dda59fffca9821d39a4, disabling compactions & flushes 2024-11-21T03:41:27,405 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:27,405 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:27,405 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. after waiting 0 ms 2024-11-21T03:41:27,405 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:27,405 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(2902): Flushing 17b90c50c9bd5dda59fffca9821d39a4 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-21T03:41:27,411 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/d582880fafdc4ff795168c9a0300be66 is 50, key is test_row_0/A:col10/1732160487207/Put/seqid=0 2024-11-21T03:41:27,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742177_1353 (size=12301) 2024-11-21T03:41:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-21T03:41:27,817 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/d582880fafdc4ff795168c9a0300be66 2024-11-21T03:41:27,831 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/5019d38de4bd44ba8926a926d139ce24 is 50, key is test_row_0/B:col10/1732160487207/Put/seqid=0 2024-11-21T03:41:27,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742178_1354 (size=12301) 2024-11-21T03:41:27,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-21T03:41:28,236 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/5019d38de4bd44ba8926a926d139ce24 2024-11-21T03:41:28,248 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/1c760359e6164a2a99909b318e48d19c is 50, key is test_row_0/C:col10/1732160487207/Put/seqid=0 2024-11-21T03:41:28,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742179_1355 (size=12301) 2024-11-21T03:41:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-21T03:41:28,652 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/1c760359e6164a2a99909b318e48d19c 2024-11-21T03:41:28,662 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/A/d582880fafdc4ff795168c9a0300be66 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/d582880fafdc4ff795168c9a0300be66 2024-11-21T03:41:28,665 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/d582880fafdc4ff795168c9a0300be66, entries=150, sequenceid=450, filesize=12.0 K 2024-11-21T03:41:28,666 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/B/5019d38de4bd44ba8926a926d139ce24 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/5019d38de4bd44ba8926a926d139ce24 2024-11-21T03:41:28,669 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/5019d38de4bd44ba8926a926d139ce24, entries=150, sequenceid=450, filesize=12.0 K 2024-11-21T03:41:28,670 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/.tmp/C/1c760359e6164a2a99909b318e48d19c as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/1c760359e6164a2a99909b318e48d19c 2024-11-21T03:41:28,673 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/1c760359e6164a2a99909b318e48d19c, entries=150, sequenceid=450, filesize=12.0 K 2024-11-21T03:41:28,674 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 17b90c50c9bd5dda59fffca9821d39a4 in 1269ms, sequenceid=450, compaction requested=false 2024-11-21T03:41:28,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/b720ee9956ab4d89a0b747c04f17cf78, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/438c360745f945e9806246b82bcb0db4, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/076171dc30354e41b5c95bdae099f047, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/d663966aa82a4c2fba608e5f9881ffb1, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/a3165b39efeb47498e8a3965aa0517d6, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/15bf8ecdfddd40fd894943a7f708faca, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/1aa53261681d4c73825ad32690e48c70, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/6067e730df0e4964a996bbf967094f19, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/39d50d4c0c6849cebd5ced4d790d0d45, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/421bec5b493748c397f68a7f34669e14, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/e708952d49bd4e0a94fea638baeabfbc, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/a707374d21304af693b346930ca55464, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/00b0b2a70dc241728556eb39622ec120, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/925eb946497141db93d01d9561c799e1, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/45705b7154194020960bd277ad28ed7a, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/d4097266e16844be883425fc50ad5cad, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/37dc7c51b515467091fb377be7f9540f, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/507e14ee630849bcbf69e59653ec1db1, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/076325d3eba845db86294fa5b2df0565, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/f7731ee868f94e20be88468c7a21881b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/6e22cca4cc2849a9bded90f4568d820b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/7699ba77261b4d6281410d12dc8c7fba, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/bc52e8d040a546c19b77c697c827a8f3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/cfb0a5a6b47f4325a6d2f91618aada03, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/a0a8fc0441fe4cfbafe07b47739d2fd3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/6e7c03da55e347c3af71032c33991ecb, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/19cdb328822449ae9f56d794868dc293, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/56130b51ca4247d498af97542083fdfc, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/91ebe45eed974291ac013929a8739ac9, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/79ef25fd007c419baf9d5e8c387a0212, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/38cc71c331ec451eb34d3ee55daadaf4, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/084afc81711d46219e3a66a8327c281d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/fb075d354c754f45a9958dc9d6b3385a] to archive 2024-11-21T03:41:28,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:41:28,677 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/b720ee9956ab4d89a0b747c04f17cf78 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/b720ee9956ab4d89a0b747c04f17cf78 2024-11-21T03:41:28,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/438c360745f945e9806246b82bcb0db4 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/438c360745f945e9806246b82bcb0db4 2024-11-21T03:41:28,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/076171dc30354e41b5c95bdae099f047 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/076171dc30354e41b5c95bdae099f047 2024-11-21T03:41:28,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/d663966aa82a4c2fba608e5f9881ffb1 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/d663966aa82a4c2fba608e5f9881ffb1 2024-11-21T03:41:28,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/a3165b39efeb47498e8a3965aa0517d6 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/a3165b39efeb47498e8a3965aa0517d6 2024-11-21T03:41:28,681 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/15bf8ecdfddd40fd894943a7f708faca to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/15bf8ecdfddd40fd894943a7f708faca 2024-11-21T03:41:28,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/1aa53261681d4c73825ad32690e48c70 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/1aa53261681d4c73825ad32690e48c70 2024-11-21T03:41:28,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/6067e730df0e4964a996bbf967094f19 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/6067e730df0e4964a996bbf967094f19 2024-11-21T03:41:28,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/39d50d4c0c6849cebd5ced4d790d0d45 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/39d50d4c0c6849cebd5ced4d790d0d45 2024-11-21T03:41:28,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/421bec5b493748c397f68a7f34669e14 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/421bec5b493748c397f68a7f34669e14 2024-11-21T03:41:28,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/e708952d49bd4e0a94fea638baeabfbc to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/e708952d49bd4e0a94fea638baeabfbc 2024-11-21T03:41:28,685 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/a707374d21304af693b346930ca55464 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/a707374d21304af693b346930ca55464 2024-11-21T03:41:28,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/00b0b2a70dc241728556eb39622ec120 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/00b0b2a70dc241728556eb39622ec120 2024-11-21T03:41:28,687 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/925eb946497141db93d01d9561c799e1 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/925eb946497141db93d01d9561c799e1 2024-11-21T03:41:28,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/45705b7154194020960bd277ad28ed7a to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/45705b7154194020960bd277ad28ed7a 2024-11-21T03:41:28,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/d4097266e16844be883425fc50ad5cad to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/d4097266e16844be883425fc50ad5cad 2024-11-21T03:41:28,689 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/37dc7c51b515467091fb377be7f9540f to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/37dc7c51b515467091fb377be7f9540f 2024-11-21T03:41:28,690 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/507e14ee630849bcbf69e59653ec1db1 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/507e14ee630849bcbf69e59653ec1db1 2024-11-21T03:41:28,690 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/076325d3eba845db86294fa5b2df0565 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/076325d3eba845db86294fa5b2df0565 2024-11-21T03:41:28,691 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/f7731ee868f94e20be88468c7a21881b to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/f7731ee868f94e20be88468c7a21881b 2024-11-21T03:41:28,692 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/6e22cca4cc2849a9bded90f4568d820b to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/6e22cca4cc2849a9bded90f4568d820b 2024-11-21T03:41:28,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/7699ba77261b4d6281410d12dc8c7fba to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/7699ba77261b4d6281410d12dc8c7fba 2024-11-21T03:41:28,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/bc52e8d040a546c19b77c697c827a8f3 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/bc52e8d040a546c19b77c697c827a8f3 2024-11-21T03:41:28,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/cfb0a5a6b47f4325a6d2f91618aada03 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/cfb0a5a6b47f4325a6d2f91618aada03 2024-11-21T03:41:28,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/a0a8fc0441fe4cfbafe07b47739d2fd3 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/a0a8fc0441fe4cfbafe07b47739d2fd3 2024-11-21T03:41:28,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/6e7c03da55e347c3af71032c33991ecb to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/6e7c03da55e347c3af71032c33991ecb 2024-11-21T03:41:28,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/19cdb328822449ae9f56d794868dc293 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/19cdb328822449ae9f56d794868dc293 2024-11-21T03:41:28,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/56130b51ca4247d498af97542083fdfc to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/56130b51ca4247d498af97542083fdfc 2024-11-21T03:41:28,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/91ebe45eed974291ac013929a8739ac9 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/91ebe45eed974291ac013929a8739ac9 2024-11-21T03:41:28,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/79ef25fd007c419baf9d5e8c387a0212 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/79ef25fd007c419baf9d5e8c387a0212 2024-11-21T03:41:28,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/38cc71c331ec451eb34d3ee55daadaf4 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/38cc71c331ec451eb34d3ee55daadaf4 2024-11-21T03:41:28,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/084afc81711d46219e3a66a8327c281d to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/084afc81711d46219e3a66a8327c281d 2024-11-21T03:41:28,701 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/fb075d354c754f45a9958dc9d6b3385a to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/fb075d354c754f45a9958dc9d6b3385a 2024-11-21T03:41:28,702 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/268b07ba64894973ac4870e0d1f7f02a, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/21057cba9305408a9965f8ecb909145b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/c4245b08c7654752a0d8f182b9524903, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/78c649a3cdbc4681889d4f4e82aaca9d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/d42af8e559a54d998bec7b6fea0d0bc3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/2c44bc16fd524c1ba34badc651cbd3ad, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/0b4e97f908b74239a55b5b9ab6af1ad7, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/d1b2f9cf850a459eb9eef3d94c893050, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/36bc9cf6e75f466da56fd339c4e53297, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/7a48199b4b9d46cfbc660aa0706dcede, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/933f62c30b7c43d09fdae28dabb0f672, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/0900ca2ae3984d4586ff31a0df434472, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/98d6c7fc9f754594b911a5d681a29b12, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/f9b6d71f5031460f8cc81d7b770873f0, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/9c2ad4bdf06b4a79a1ea648b2a350878, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/09613b6a77fa4ed6bcaacbc2a5d5d4ca, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/3bb6c88be6704b2990eddcd98a26da5d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/f2b48261f33a4037ba935179ddc105d5, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/506088fcd7074c4a9d408df5b07c5dd5, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/2740747bfe1c40a684c1ae6d46d2e78b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/cbbc65f18a9c4e878f6cd1b1b22a8235, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/c1bbfd6134ee4838bef41341c8e6166e, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/47d9beef858d455d87833a08e8bd9177, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/dbfd34aef80545efadcb5d1d5bd39b9e, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/bb1c3213f28c45debab044b3562fbfff, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/b4b34c62c3d94cbeb63a239290088f3c, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/0d71a05f59e14364b990da73795d4315, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/93b35942b1ec4a8e9aaaeda8f2871b71, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/699e0033110541bb9d2195544291e011, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/7f65f6bb4ee04d06a1cc518dcdd1854d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/41c6accb2e204f6f9e33b6a84f2fd103, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/c5eb5ae95caf41cf88f10a64b197b94b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/51cb6c37fc134d4da781d0c579dde579] to archive 2024-11-21T03:41:28,703 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:41:28,704 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/268b07ba64894973ac4870e0d1f7f02a to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/268b07ba64894973ac4870e0d1f7f02a 2024-11-21T03:41:28,705 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/21057cba9305408a9965f8ecb909145b to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/21057cba9305408a9965f8ecb909145b 2024-11-21T03:41:28,706 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/c4245b08c7654752a0d8f182b9524903 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/c4245b08c7654752a0d8f182b9524903 2024-11-21T03:41:28,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/78c649a3cdbc4681889d4f4e82aaca9d to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/78c649a3cdbc4681889d4f4e82aaca9d 2024-11-21T03:41:28,707 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/d42af8e559a54d998bec7b6fea0d0bc3 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/d42af8e559a54d998bec7b6fea0d0bc3 2024-11-21T03:41:28,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/2c44bc16fd524c1ba34badc651cbd3ad to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/2c44bc16fd524c1ba34badc651cbd3ad 2024-11-21T03:41:28,709 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/0b4e97f908b74239a55b5b9ab6af1ad7 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/0b4e97f908b74239a55b5b9ab6af1ad7 2024-11-21T03:41:28,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/d1b2f9cf850a459eb9eef3d94c893050 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/d1b2f9cf850a459eb9eef3d94c893050 2024-11-21T03:41:28,710 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/36bc9cf6e75f466da56fd339c4e53297 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/36bc9cf6e75f466da56fd339c4e53297 2024-11-21T03:41:28,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/7a48199b4b9d46cfbc660aa0706dcede to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/7a48199b4b9d46cfbc660aa0706dcede 2024-11-21T03:41:28,712 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/933f62c30b7c43d09fdae28dabb0f672 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/933f62c30b7c43d09fdae28dabb0f672 2024-11-21T03:41:28,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/0900ca2ae3984d4586ff31a0df434472 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/0900ca2ae3984d4586ff31a0df434472 2024-11-21T03:41:28,713 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/98d6c7fc9f754594b911a5d681a29b12 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/98d6c7fc9f754594b911a5d681a29b12 2024-11-21T03:41:28,714 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/f9b6d71f5031460f8cc81d7b770873f0 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/f9b6d71f5031460f8cc81d7b770873f0 2024-11-21T03:41:28,715 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/9c2ad4bdf06b4a79a1ea648b2a350878 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/9c2ad4bdf06b4a79a1ea648b2a350878 2024-11-21T03:41:28,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/09613b6a77fa4ed6bcaacbc2a5d5d4ca to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/09613b6a77fa4ed6bcaacbc2a5d5d4ca 2024-11-21T03:41:28,717 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/3bb6c88be6704b2990eddcd98a26da5d to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/3bb6c88be6704b2990eddcd98a26da5d 2024-11-21T03:41:28,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/f2b48261f33a4037ba935179ddc105d5 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/f2b48261f33a4037ba935179ddc105d5 2024-11-21T03:41:28,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/506088fcd7074c4a9d408df5b07c5dd5 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/506088fcd7074c4a9d408df5b07c5dd5 2024-11-21T03:41:28,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/2740747bfe1c40a684c1ae6d46d2e78b to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/2740747bfe1c40a684c1ae6d46d2e78b 2024-11-21T03:41:28,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/cbbc65f18a9c4e878f6cd1b1b22a8235 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/cbbc65f18a9c4e878f6cd1b1b22a8235 2024-11-21T03:41:28,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/c1bbfd6134ee4838bef41341c8e6166e to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/c1bbfd6134ee4838bef41341c8e6166e 2024-11-21T03:41:28,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/47d9beef858d455d87833a08e8bd9177 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/47d9beef858d455d87833a08e8bd9177 2024-11-21T03:41:28,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/dbfd34aef80545efadcb5d1d5bd39b9e to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/dbfd34aef80545efadcb5d1d5bd39b9e 2024-11-21T03:41:28,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/bb1c3213f28c45debab044b3562fbfff to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/bb1c3213f28c45debab044b3562fbfff 2024-11-21T03:41:28,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/b4b34c62c3d94cbeb63a239290088f3c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/b4b34c62c3d94cbeb63a239290088f3c 2024-11-21T03:41:28,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/0d71a05f59e14364b990da73795d4315 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/0d71a05f59e14364b990da73795d4315 2024-11-21T03:41:28,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/93b35942b1ec4a8e9aaaeda8f2871b71 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/93b35942b1ec4a8e9aaaeda8f2871b71 2024-11-21T03:41:28,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/699e0033110541bb9d2195544291e011 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/699e0033110541bb9d2195544291e011 2024-11-21T03:41:28,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/7f65f6bb4ee04d06a1cc518dcdd1854d to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/7f65f6bb4ee04d06a1cc518dcdd1854d 2024-11-21T03:41:28,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/41c6accb2e204f6f9e33b6a84f2fd103 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/41c6accb2e204f6f9e33b6a84f2fd103 2024-11-21T03:41:28,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/c5eb5ae95caf41cf88f10a64b197b94b to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/c5eb5ae95caf41cf88f10a64b197b94b 2024-11-21T03:41:28,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/51cb6c37fc134d4da781d0c579dde579 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/51cb6c37fc134d4da781d0c579dde579 2024-11-21T03:41:28,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/2443b8629d6f40faa074c022a2ccfe2b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/7ed7e241f59740be94f449da7f633665, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/5f15c7cc40c14033a8a44617bb9959b9, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/20c792e5886548ee94c342071a58cb22, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/ebd2ac6e8661449c885f3b19e3eb1439, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/78b8f35be62b43daaaf59c0d40a7f59a, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/545e3696f5a643e3987e6d548e2cf6d4, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/4cfcc5d8342a42aea90b09d3c2f68c99, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/8b250df71c7a40c8867819998d54d39f, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/cd5abd61b862448db6312dce70caccb1, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/2f4bba879c5a4df5acfd43c92f3d2577, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/ce8e72186ada4ed997c6db144ed54126, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/5481b94b6912484fa6ca71d0fc8ab6df, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/810c5394dada4367a399bb7dc4a314c1, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/a1d94041d078465585092aea0e6a11a4, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/270766ef70ae472388cbc574290f886a, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/94492f1e5971457a846e274d1c0b36d2, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/c51c4282209445bbbec0b7ce132f4e8f, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/46acc9d8ae7f470b93be56332e068576, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/9ef45713d45a4b788149545c2d0dfed9, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/06a38cd506d743368527099181b3447e, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/f070e7b2248e46fca3aedaa60fe0ad65, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/0ef6a856d7a04b7d94eda1ceb5aff7d7, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/81de1ad7a9eb4953ba28f4db245527f2, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/871aee8449554f29914f45a3f859f0aa, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/4fdb99acae984466bc4e8d89af796876, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/9b6fe42d98ee4fd694908188f9521239, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/1ce084bde8a841ea8727315ebf4ff28f, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/764166e092474e50af35a5edfc831c2d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/466116c279494b429efde4837d361bdc, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/9e54796d0b8f4dd5aa80aa3bfb39429a, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/6b6be39e6994483abf8ab51dec23f971, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/cfab43ca85a741358d2f390638d183dd] to archive 2024-11-21T03:41:28,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:41:28,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/2443b8629d6f40faa074c022a2ccfe2b to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/2443b8629d6f40faa074c022a2ccfe2b 2024-11-21T03:41:28,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/7ed7e241f59740be94f449da7f633665 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/7ed7e241f59740be94f449da7f633665 2024-11-21T03:41:28,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/5f15c7cc40c14033a8a44617bb9959b9 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/5f15c7cc40c14033a8a44617bb9959b9 2024-11-21T03:41:28,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/20c792e5886548ee94c342071a58cb22 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/20c792e5886548ee94c342071a58cb22 2024-11-21T03:41:28,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/ebd2ac6e8661449c885f3b19e3eb1439 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/ebd2ac6e8661449c885f3b19e3eb1439 2024-11-21T03:41:28,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/78b8f35be62b43daaaf59c0d40a7f59a to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/78b8f35be62b43daaaf59c0d40a7f59a 2024-11-21T03:41:28,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/545e3696f5a643e3987e6d548e2cf6d4 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/545e3696f5a643e3987e6d548e2cf6d4 2024-11-21T03:41:28,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/4cfcc5d8342a42aea90b09d3c2f68c99 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/4cfcc5d8342a42aea90b09d3c2f68c99 2024-11-21T03:41:28,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/8b250df71c7a40c8867819998d54d39f to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/8b250df71c7a40c8867819998d54d39f 2024-11-21T03:41:28,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/cd5abd61b862448db6312dce70caccb1 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/cd5abd61b862448db6312dce70caccb1 2024-11-21T03:41:28,744 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/2f4bba879c5a4df5acfd43c92f3d2577 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/2f4bba879c5a4df5acfd43c92f3d2577 2024-11-21T03:41:28,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/ce8e72186ada4ed997c6db144ed54126 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/ce8e72186ada4ed997c6db144ed54126 2024-11-21T03:41:28,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/5481b94b6912484fa6ca71d0fc8ab6df to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/5481b94b6912484fa6ca71d0fc8ab6df 2024-11-21T03:41:28,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/810c5394dada4367a399bb7dc4a314c1 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/810c5394dada4367a399bb7dc4a314c1 2024-11-21T03:41:28,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/a1d94041d078465585092aea0e6a11a4 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/a1d94041d078465585092aea0e6a11a4 2024-11-21T03:41:28,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/270766ef70ae472388cbc574290f886a to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/270766ef70ae472388cbc574290f886a 2024-11-21T03:41:28,750 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/94492f1e5971457a846e274d1c0b36d2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/94492f1e5971457a846e274d1c0b36d2 2024-11-21T03:41:28,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/c51c4282209445bbbec0b7ce132f4e8f to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/c51c4282209445bbbec0b7ce132f4e8f 2024-11-21T03:41:28,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/46acc9d8ae7f470b93be56332e068576 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/46acc9d8ae7f470b93be56332e068576 2024-11-21T03:41:28,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/9ef45713d45a4b788149545c2d0dfed9 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/9ef45713d45a4b788149545c2d0dfed9 2024-11-21T03:41:28,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/06a38cd506d743368527099181b3447e to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/06a38cd506d743368527099181b3447e 2024-11-21T03:41:28,755 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/f070e7b2248e46fca3aedaa60fe0ad65 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/f070e7b2248e46fca3aedaa60fe0ad65 2024-11-21T03:41:28,756 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/0ef6a856d7a04b7d94eda1ceb5aff7d7 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/0ef6a856d7a04b7d94eda1ceb5aff7d7 2024-11-21T03:41:28,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/81de1ad7a9eb4953ba28f4db245527f2 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/81de1ad7a9eb4953ba28f4db245527f2 2024-11-21T03:41:28,758 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/871aee8449554f29914f45a3f859f0aa to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/871aee8449554f29914f45a3f859f0aa 2024-11-21T03:41:28,759 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/4fdb99acae984466bc4e8d89af796876 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/4fdb99acae984466bc4e8d89af796876 2024-11-21T03:41:28,761 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/9b6fe42d98ee4fd694908188f9521239 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/9b6fe42d98ee4fd694908188f9521239 2024-11-21T03:41:28,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/1ce084bde8a841ea8727315ebf4ff28f to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/1ce084bde8a841ea8727315ebf4ff28f 2024-11-21T03:41:28,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/764166e092474e50af35a5edfc831c2d to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/764166e092474e50af35a5edfc831c2d 2024-11-21T03:41:28,764 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/466116c279494b429efde4837d361bdc to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/466116c279494b429efde4837d361bdc 2024-11-21T03:41:28,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/9e54796d0b8f4dd5aa80aa3bfb39429a to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/9e54796d0b8f4dd5aa80aa3bfb39429a 2024-11-21T03:41:28,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/6b6be39e6994483abf8ab51dec23f971 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/6b6be39e6994483abf8ab51dec23f971 2024-11-21T03:41:28,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/cfab43ca85a741358d2f390638d183dd to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/cfab43ca85a741358d2f390638d183dd 2024-11-21T03:41:28,770 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/recovered.edits/453.seqid, newMaxSeqId=453, maxSeqId=1 2024-11-21T03:41:28,771 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. 2024-11-21T03:41:28,771 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for 17b90c50c9bd5dda59fffca9821d39a4: Waiting for close lock at 1732160487405Running coprocessor pre-close hooks at 1732160487405Disabling compacts and flushes for region at 1732160487405Disabling writes for close at 1732160487405Obtaining lock to block concurrent updates at 1732160487405Preparing flush snapshotting stores in 17b90c50c9bd5dda59fffca9821d39a4 at 1732160487405Finished memstore snapshotting TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1732160487406 (+1 ms)Flushing stores of TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4. at 1732160487406Flushing 17b90c50c9bd5dda59fffca9821d39a4/A: creating writer at 1732160487406Flushing 17b90c50c9bd5dda59fffca9821d39a4/A: appending metadata at 1732160487410 (+4 ms)Flushing 17b90c50c9bd5dda59fffca9821d39a4/A: closing flushed file at 1732160487410Flushing 17b90c50c9bd5dda59fffca9821d39a4/B: creating writer at 1732160487826 (+416 ms)Flushing 17b90c50c9bd5dda59fffca9821d39a4/B: appending metadata at 1732160487830 (+4 ms)Flushing 17b90c50c9bd5dda59fffca9821d39a4/B: closing flushed file at 1732160487831 (+1 ms)Flushing 17b90c50c9bd5dda59fffca9821d39a4/C: creating writer at 1732160488245 (+414 ms)Flushing 17b90c50c9bd5dda59fffca9821d39a4/C: appending metadata at 1732160488247 (+2 ms)Flushing 17b90c50c9bd5dda59fffca9821d39a4/C: closing flushed file at 1732160488247Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@639ed689: reopening flushed file at 1732160488661 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69cf2a5b: reopening flushed file at 1732160488666 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b5c5053: reopening flushed file at 1732160488669 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 17b90c50c9bd5dda59fffca9821d39a4 in 1269ms, sequenceid=450, compaction requested=false at 1732160488674 (+5 ms)Writing region close event to WAL at 1732160488768 (+94 ms)Running coprocessor post-close hooks at 1732160488771 (+3 ms)Closed at 1732160488771 2024-11-21T03:41:28,773 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed 17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:41:28,774 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=17b90c50c9bd5dda59fffca9821d39a4, regionState=CLOSED 2024-11-21T03:41:28,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 because future has completed 2024-11-21T03:41:28,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-21T03:41:28,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure 17b90c50c9bd5dda59fffca9821d39a4, server=fdc0a393186c,34631,1732160396889 in 1.5240 sec 2024-11-21T03:41:28,778 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-21T03:41:28,778 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17b90c50c9bd5dda59fffca9821d39a4, UNASSIGN in 1.5290 sec 2024-11-21T03:41:28,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-21T03:41:28,781 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5330 sec 2024-11-21T03:41:28,782 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732160488781"}]},"ts":"1732160488781"} 2024-11-21T03:41:28,783 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T03:41:28,783 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T03:41:28,785 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5430 sec 2024-11-21T03:41:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-21T03:41:29,388 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:41:29,389 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T03:41:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:41:29,392 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=96, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:41:29,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-21T03:41:29,393 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=96, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:41:29,396 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:41:29,401 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A, FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B, FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C, FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/recovered.edits] 2024-11-21T03:41:29,406 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/cfd918622c14418787aa2c5899d377ef to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/cfd918622c14418787aa2c5899d377ef 2024-11-21T03:41:29,408 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/d582880fafdc4ff795168c9a0300be66 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/A/d582880fafdc4ff795168c9a0300be66 2024-11-21T03:41:29,412 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/3139c907a8bc471482de9065d8a793d3 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/3139c907a8bc471482de9065d8a793d3 2024-11-21T03:41:29,414 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/5019d38de4bd44ba8926a926d139ce24 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/B/5019d38de4bd44ba8926a926d139ce24 2024-11-21T03:41:29,416 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/1c760359e6164a2a99909b318e48d19c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/1c760359e6164a2a99909b318e48d19c 2024-11-21T03:41:29,417 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/608d5df4fd6643518323001d29bebf95 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/C/608d5df4fd6643518323001d29bebf95 2024-11-21T03:41:29,420 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/recovered.edits/453.seqid to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4/recovered.edits/453.seqid 2024-11-21T03:41:29,420 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/17b90c50c9bd5dda59fffca9821d39a4 2024-11-21T03:41:29,420 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T03:41:29,422 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=96, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:41:29,425 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T03:41:29,427 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T03:41:29,428 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=96, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:41:29,428 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T03:41:29,428 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732160489428"}]},"ts":"9223372036854775807"} 2024-11-21T03:41:29,430 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T03:41:29,430 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 17b90c50c9bd5dda59fffca9821d39a4, NAME => 'TestAcidGuarantees,,1732160455512.17b90c50c9bd5dda59fffca9821d39a4.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T03:41:29,430 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T03:41:29,430 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732160489430"}]},"ts":"9223372036854775807"} 2024-11-21T03:41:29,431 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T03:41:29,432 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=96, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:41:29,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 43 msec 2024-11-21T03:41:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-21T03:41:29,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T03:41:29,501 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:41:29,512 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238 (was 242), OpenFileDescriptor=445 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=491 (was 613), ProcessCount=11 (was 11), AvailableMemoryMB=6594 (was 6373) - AvailableMemoryMB LEAK? - 2024-11-21T03:41:29,523 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=491, ProcessCount=11, AvailableMemoryMB=6594 2024-11-21T03:41:29,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T03:41:29,525 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T03:41:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T03:41:29,527 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T03:41:29,527 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:29,527 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 97 2024-11-21T03:41:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T03:41:29,528 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T03:41:29,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742180_1356 (size=959) 2024-11-21T03:41:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T03:41:29,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T03:41:29,936 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0 2024-11-21T03:41:29,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742181_1357 (size=53) 2024-11-21T03:41:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T03:41:30,345 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:41:30,345 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f60eea9fbbad1295666bbf6cb6ce080c, disabling compactions & flushes 2024-11-21T03:41:30,345 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:30,345 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:30,345 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. after waiting 0 ms 2024-11-21T03:41:30,345 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:30,345 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:30,345 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f60eea9fbbad1295666bbf6cb6ce080c: Waiting for close lock at 1732160490345Disabling compacts and flushes for region at 1732160490345Disabling writes for close at 1732160490345Writing region close event to WAL at 1732160490345Closed at 1732160490345 2024-11-21T03:41:30,347 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T03:41:30,347 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732160490347"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732160490347"}]},"ts":"1732160490347"} 2024-11-21T03:41:30,349 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T03:41:30,351 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T03:41:30,351 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732160490351"}]},"ts":"1732160490351"} 2024-11-21T03:41:30,353 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T03:41:30,354 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f60eea9fbbad1295666bbf6cb6ce080c, ASSIGN}] 2024-11-21T03:41:30,355 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f60eea9fbbad1295666bbf6cb6ce080c, ASSIGN 2024-11-21T03:41:30,356 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f60eea9fbbad1295666bbf6cb6ce080c, ASSIGN; state=OFFLINE, location=fdc0a393186c,34631,1732160396889; forceNewPlan=false, retain=false 2024-11-21T03:41:30,507 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=f60eea9fbbad1295666bbf6cb6ce080c, regionState=OPENING, regionLocation=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:30,509 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f60eea9fbbad1295666bbf6cb6ce080c, ASSIGN because future has completed 2024-11-21T03:41:30,509 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889}] 2024-11-21T03:41:30,665 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:30,665 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7752): Opening region: {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:41:30,666 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:30,666 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:41:30,666 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7794): checking encryption for f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:30,666 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(7797): checking classloading for f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T03:41:30,668 INFO [StoreOpener-f60eea9fbbad1295666bbf6cb6ce080c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:30,670 INFO [StoreOpener-f60eea9fbbad1295666bbf6cb6ce080c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f60eea9fbbad1295666bbf6cb6ce080c columnFamilyName A 2024-11-21T03:41:30,670 DEBUG [StoreOpener-f60eea9fbbad1295666bbf6cb6ce080c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:30,671 INFO [StoreOpener-f60eea9fbbad1295666bbf6cb6ce080c-1 {}] regionserver.HStore(327): Store=f60eea9fbbad1295666bbf6cb6ce080c/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:41:30,671 INFO [StoreOpener-f60eea9fbbad1295666bbf6cb6ce080c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:30,673 INFO [StoreOpener-f60eea9fbbad1295666bbf6cb6ce080c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f60eea9fbbad1295666bbf6cb6ce080c columnFamilyName B 2024-11-21T03:41:30,673 DEBUG [StoreOpener-f60eea9fbbad1295666bbf6cb6ce080c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:30,673 INFO [StoreOpener-f60eea9fbbad1295666bbf6cb6ce080c-1 {}] regionserver.HStore(327): Store=f60eea9fbbad1295666bbf6cb6ce080c/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:41:30,673 INFO [StoreOpener-f60eea9fbbad1295666bbf6cb6ce080c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:30,675 INFO [StoreOpener-f60eea9fbbad1295666bbf6cb6ce080c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f60eea9fbbad1295666bbf6cb6ce080c columnFamilyName C 2024-11-21T03:41:30,675 DEBUG [StoreOpener-f60eea9fbbad1295666bbf6cb6ce080c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:30,675 INFO [StoreOpener-f60eea9fbbad1295666bbf6cb6ce080c-1 {}] regionserver.HStore(327): Store=f60eea9fbbad1295666bbf6cb6ce080c/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:41:30,675 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1038): replaying wal for f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:30,676 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:30,677 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:30,678 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1048): stopping wal replay for f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:30,678 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1060): Cleaning up temporary data for f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:30,678 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T03:41:30,680 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1093): writing seq id for f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:30,682 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T03:41:30,683 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1114): Opened f60eea9fbbad1295666bbf6cb6ce080c; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68313500, jitterRate=0.017950475215911865}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T03:41:30,683 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:30,684 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegion(1006): Region open journal for f60eea9fbbad1295666bbf6cb6ce080c: Running coprocessor pre-open hook at 1732160490666Writing region info on filesystem at 1732160490666Initializing all the Stores at 1732160490667 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732160490667Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732160490668 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732160490668Cleaning up temporary data from old regions at 1732160490678 (+10 ms)Running coprocessor post-open hooks at 1732160490683 (+5 ms)Region opened successfully at 1732160490684 (+1 ms) 2024-11-21T03:41:30,685 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., pid=99, masterSystemTime=1732160490660 2024-11-21T03:41:30,687 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:30,687 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=99}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:30,688 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=f60eea9fbbad1295666bbf6cb6ce080c, regionState=OPEN, openSeqNum=2, regionLocation=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:30,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 because future has completed 2024-11-21T03:41:30,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-21T03:41:30,692 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 in 181 msec 2024-11-21T03:41:30,693 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-21T03:41:30,693 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f60eea9fbbad1295666bbf6cb6ce080c, ASSIGN in 339 msec 2024-11-21T03:41:30,694 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T03:41:30,694 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732160490694"}]},"ts":"1732160490694"} 2024-11-21T03:41:30,695 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T03:41:30,696 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=97, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T03:41:30,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1710 sec 2024-11-21T03:41:31,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T03:41:31,678 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:41:31,679 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283b5885, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:31,679 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:41:31,679 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:41:31,680 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:41:31,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:41:31,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:41:31,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a5cf73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:31,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:41:31,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:41:31,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:31,683 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:41:31,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eb36ad0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:31,684 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:41:31,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:41:31,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:41:31,688 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:41:31,689 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T03:41:31,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is fdc0a393186c,44285,1732160396080 2024-11-21T03:41:31,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@144533de 2024-11-21T03:41:31,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T03:41:31,691 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50898, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T03:41:31,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T03:41:31,693 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T03:41:31,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=100, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-21T03:41:31,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742182_1358 (size=995) 2024-11-21T03:41:32,139 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-21T03:41:32,139 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-21T03:41:32,144 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T03:41:32,150 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f60eea9fbbad1295666bbf6cb6ce080c, REOPEN/MOVE}] 2024-11-21T03:41:32,151 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f60eea9fbbad1295666bbf6cb6ce080c, REOPEN/MOVE 2024-11-21T03:41:32,152 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=f60eea9fbbad1295666bbf6cb6ce080c, regionState=CLOSING, regionLocation=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:32,153 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f60eea9fbbad1295666bbf6cb6ce080c, REOPEN/MOVE because future has completed 2024-11-21T03:41:32,153 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T03:41:32,153 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE, hasLock=false; CloseRegionProcedure f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889}] 2024-11-21T03:41:32,305 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(122): Close f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:32,305 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T03:41:32,305 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1722): Closing f60eea9fbbad1295666bbf6cb6ce080c, disabling compactions & flushes 2024-11-21T03:41:32,305 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:32,305 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:32,305 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. after waiting 0 ms 2024-11-21T03:41:32,305 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:32,308 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T03:41:32,309 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:32,309 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegion(1676): Region close journal for f60eea9fbbad1295666bbf6cb6ce080c: Waiting for close lock at 1732160492305Running coprocessor pre-close hooks at 1732160492305Disabling compacts and flushes for region at 1732160492305Disabling writes for close at 1732160492305Writing region close event to WAL at 1732160492305Running coprocessor post-close hooks at 1732160492309 (+4 ms)Closed at 1732160492309 2024-11-21T03:41:32,309 WARN [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] regionserver.HRegionServer(3299): Not adding moved region record: f60eea9fbbad1295666bbf6cb6ce080c to self. 2024-11-21T03:41:32,310 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=103}] handler.UnassignRegionHandler(157): Closed f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:32,311 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=f60eea9fbbad1295666bbf6cb6ce080c, regionState=CLOSED 2024-11-21T03:41:32,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE, hasLock=false; CloseRegionProcedure f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 because future has completed 2024-11-21T03:41:32,315 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-21T03:41:32,315 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; CloseRegionProcedure f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 in 160 msec 2024-11-21T03:41:32,315 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f60eea9fbbad1295666bbf6cb6ce080c, REOPEN/MOVE; state=CLOSED, location=fdc0a393186c,34631,1732160396889; forceNewPlan=false, retain=true 2024-11-21T03:41:32,466 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=f60eea9fbbad1295666bbf6cb6ce080c, regionState=OPENING, regionLocation=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:32,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f60eea9fbbad1295666bbf6cb6ce080c, REOPEN/MOVE because future has completed 2024-11-21T03:41:32,470 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889}] 2024-11-21T03:41:32,626 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:32,626 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7752): Opening region: {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:41:32,626 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:32,626 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:41:32,626 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7794): checking encryption for f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:32,626 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(7797): checking classloading for f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:32,628 INFO [StoreOpener-f60eea9fbbad1295666bbf6cb6ce080c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:32,628 INFO [StoreOpener-f60eea9fbbad1295666bbf6cb6ce080c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f60eea9fbbad1295666bbf6cb6ce080c columnFamilyName A 2024-11-21T03:41:32,629 DEBUG [StoreOpener-f60eea9fbbad1295666bbf6cb6ce080c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:32,630 INFO [StoreOpener-f60eea9fbbad1295666bbf6cb6ce080c-1 {}] regionserver.HStore(327): Store=f60eea9fbbad1295666bbf6cb6ce080c/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:41:32,630 INFO [StoreOpener-f60eea9fbbad1295666bbf6cb6ce080c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:32,631 INFO [StoreOpener-f60eea9fbbad1295666bbf6cb6ce080c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f60eea9fbbad1295666bbf6cb6ce080c columnFamilyName B 2024-11-21T03:41:32,631 DEBUG [StoreOpener-f60eea9fbbad1295666bbf6cb6ce080c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:32,631 INFO [StoreOpener-f60eea9fbbad1295666bbf6cb6ce080c-1 {}] regionserver.HStore(327): Store=f60eea9fbbad1295666bbf6cb6ce080c/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:41:32,631 INFO [StoreOpener-f60eea9fbbad1295666bbf6cb6ce080c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:32,632 INFO [StoreOpener-f60eea9fbbad1295666bbf6cb6ce080c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f60eea9fbbad1295666bbf6cb6ce080c columnFamilyName C 2024-11-21T03:41:32,632 DEBUG [StoreOpener-f60eea9fbbad1295666bbf6cb6ce080c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:32,632 INFO [StoreOpener-f60eea9fbbad1295666bbf6cb6ce080c-1 {}] regionserver.HStore(327): Store=f60eea9fbbad1295666bbf6cb6ce080c/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:41:32,632 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1038): replaying wal for f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:32,633 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:32,634 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:32,635 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1048): stopping wal replay for f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:32,635 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1060): Cleaning up temporary data for f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:32,636 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T03:41:32,637 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1093): writing seq id for f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:32,638 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1114): Opened f60eea9fbbad1295666bbf6cb6ce080c; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63553295, jitterRate=-0.05298210680484772}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T03:41:32,638 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:32,639 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegion(1006): Region open journal for f60eea9fbbad1295666bbf6cb6ce080c: Running coprocessor pre-open hook at 1732160492626Writing region info on filesystem at 1732160492626Initializing all the Stores at 1732160492627 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732160492627Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732160492627Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732160492628 (+1 ms)Cleaning up temporary data from old regions at 1732160492635 (+7 ms)Running coprocessor post-open hooks at 1732160492638 (+3 ms)Region opened successfully at 1732160492639 (+1 ms) 2024-11-21T03:41:32,640 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., pid=104, masterSystemTime=1732160492623 2024-11-21T03:41:32,642 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:32,642 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=104}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:32,643 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=102 updating hbase:meta row=f60eea9fbbad1295666bbf6cb6ce080c, regionState=OPEN, openSeqNum=5, regionLocation=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:32,644 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=102, state=RUNNABLE, hasLock=false; OpenRegionProcedure f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 because future has completed 2024-11-21T03:41:32,647 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=102 2024-11-21T03:41:32,647 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=102, state=SUCCESS, hasLock=false; OpenRegionProcedure f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 in 176 msec 2024-11-21T03:41:32,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-21T03:41:32,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f60eea9fbbad1295666bbf6cb6ce080c, REOPEN/MOVE in 497 msec 2024-11-21T03:41:32,652 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-21T03:41:32,652 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 506 msec 2024-11-21T03:41:32,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 958 msec 2024-11-21T03:41:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100 2024-11-21T03:41:32,655 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:41:32,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,655 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:41:32,656 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:41:32,656 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:41:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:41:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:41:32,656 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47139b05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:41:32,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:41:32,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:32,657 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:41:32,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d0e472f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,658 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:41:32,658 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:41:32,659 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:41:32,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:41:32,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:41:32,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70ca94f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:41:32,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:41:32,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:32,660 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:41:32,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cfac1eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,661 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:41:32,661 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:41:32,661 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:41:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:41:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:41:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@401f143, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:41:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:41:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:32,663 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58640, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:41:32,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22d9ab5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345722e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,664 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:41:32,664 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:41:32,664 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:41:32,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:41:32,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:41:32,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5eac852d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:41:32,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:41:32,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:32,665 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:41:32,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b6aa60a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd3a90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:41:32,666 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:41:32,666 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:41:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:41:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:41:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44cdef66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:41:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:41:32,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:32,668 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:41:32,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21565ad7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eba3e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,670 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:41:32,670 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:41:32,670 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:41:32,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:41:32,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:41:32,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@439cd940, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:41:32,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:41:32,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:32,671 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:41:32,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6021e6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79570776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,672 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:41:32,672 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:41:32,673 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:41:32,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:41:32,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:41:32,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e08d069, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:41:32,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:41:32,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:32,674 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58692, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:41:32,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a4347bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43065a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,675 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:41:32,675 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:41:32,676 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:41:32,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:41:32,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:41:32,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6de54d24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:41:32,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:41:32,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:32,677 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:41:32,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c122365, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bcd92c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,678 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:41:32,678 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:41:32,678 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:41:32,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:41:32,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:41:32,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8c6002d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:41:32,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:41:32,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:32,680 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58734, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:41:32,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50b70dd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,681 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b779047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,681 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:41:32,681 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:41:32,682 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:41:32,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:41:32,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:41:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18a4f8c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:41:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:41:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:32,684 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:41:32,684 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@443b608c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:41:32,685 DEBUG [Thread-1635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:41:32,685 DEBUG [Thread-1624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:41:32,685 DEBUG [Thread-1637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:41:32,685 DEBUG [Thread-1626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:41:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:41:32,686 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:41:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:41:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:41:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:41:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:41:32,686 DEBUG [Thread-1629 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:41:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:41:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:41:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:41:32,686 DEBUG [Thread-1633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:41:32,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-21T03:41:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:41:32,688 DEBUG [Thread-1618 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:41:32,688 DEBUG [Thread-1620 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:41:32,688 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:41:32,688 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:41:32,688 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:41:32,688 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:41:32,688 DEBUG [Thread-1622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:41:32,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:41:32,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T03:41:32,689 DEBUG [Thread-1631 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:41:32,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:41:32,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:41:32,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:41:32,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:41:32,689 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:41:32,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:41:32,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:41:32,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:41:32,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:41:32,690 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:41:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:41:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5] 2024-11-21T03:41:32,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5] 2024-11-21T03:41:32,691 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:41:32,691 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:41:32,692 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:41:32,692 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37306, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:41:32,692 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:41:32,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5] 2024-11-21T03:41:32,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5] 2024-11-21T03:41:32,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5] 2024-11-21T03:41:32,694 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:41:32,694 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:41:32,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5] 2024-11-21T03:41:32,695 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37318, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:41:32,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5] 2024-11-21T03:41:32,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5] 2024-11-21T03:41:32,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5] 2024-11-21T03:41:32,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5] 2024-11-21T03:41:32,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:32,703 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:41:32,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f60eea9fbbad1295666bbf6cb6ce080c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:41:32,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210c38e1ef632649ba9865781066df12fc_f60eea9fbbad1295666bbf6cb6ce080c is 50, key is test_row_0/A:col10/1732160492701/Put/seqid=0 2024-11-21T03:41:32,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742183_1359 (size=12154) 2024-11-21T03:41:32,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:32,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160552737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:32,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:32,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:32,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160552744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:32,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:32,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:32,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:32,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:32,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160552744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:32,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:32,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:32,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:32,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:32,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160552745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:32,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:32,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:32,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:32,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:32,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160552746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:32,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:32,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:32,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T03:41:32,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T03:41:32,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:32,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:32,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:32,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:32,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:32,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:32,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:32,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160552857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:32,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:32,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:32,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:32,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:32,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160552857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:32,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:32,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:32,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:32,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:32,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160552857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:32,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:32,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:32,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:32,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:32,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160552857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:32,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:32,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:32,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160552857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:32,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:32,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:32,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:32,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:32,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:33,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T03:41:33,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:33,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:33,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:33,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:33,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T03:41:33,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:33,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160553067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:33,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:33,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160553067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:33,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:33,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160553067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:33,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160553067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:33,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160553076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:33,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:33,142 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:33,145 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210c38e1ef632649ba9865781066df12fc_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210c38e1ef632649ba9865781066df12fc_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:33,146 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/642b47f67c1548cc842addaa323cd83c, store: [table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:33,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/642b47f67c1548cc842addaa323cd83c is 175, key is test_row_0/A:col10/1732160492701/Put/seqid=0 2024-11-21T03:41:33,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742184_1360 (size=30955) 2024-11-21T03:41:33,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T03:41:33,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:33,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:33,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:33,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:33,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:33,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:33,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T03:41:33,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:33,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:33,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:33,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:33,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:33,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:33,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T03:41:33,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:33,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160553377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:33,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:33,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:33,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160553377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:33,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:33,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:33,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160553377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:33,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160553377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:33,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:33,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160553386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:33,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,392 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:33,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T03:41:33,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:33,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:33,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:33,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:33,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:33,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:33,550 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/642b47f67c1548cc842addaa323cd83c 2024-11-21T03:41:33,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/c6a524c26efb456b91ede14a446ab1da is 50, key is test_row_0/B:col10/1732160492701/Put/seqid=0 2024-11-21T03:41:33,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742185_1361 (size=12001) 2024-11-21T03:41:33,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T03:41:33,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:33,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:33,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:33,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:33,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:33,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T03:41:33,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:33,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:33,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:33,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:33,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:33,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:33,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T03:41:33,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:33,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160553887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:33,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:33,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160553887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:33,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:33,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:33,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160553896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:33,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160553896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:33,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160553897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:33,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T03:41:33,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:33,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:33,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:33,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:33,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:33,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:33,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/c6a524c26efb456b91ede14a446ab1da 2024-11-21T03:41:33,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/c535eadae0ee47e2be546c5c9f86a669 is 50, key is test_row_0/C:col10/1732160492701/Put/seqid=0 2024-11-21T03:41:34,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742186_1362 (size=12001) 2024-11-21T03:41:34,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T03:41:34,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:34,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:34,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:34,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:34,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:34,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:34,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T03:41:34,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:34,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:34,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:34,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:34,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:34,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:34,300 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T03:41:34,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T03:41:34,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:34,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:34,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:34,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:34,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:34,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:34,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/c535eadae0ee47e2be546c5c9f86a669 2024-11-21T03:41:34,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/642b47f67c1548cc842addaa323cd83c as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/642b47f67c1548cc842addaa323cd83c 2024-11-21T03:41:34,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/642b47f67c1548cc842addaa323cd83c, entries=150, sequenceid=16, filesize=30.2 K 2024-11-21T03:41:34,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/c6a524c26efb456b91ede14a446ab1da as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/c6a524c26efb456b91ede14a446ab1da 2024-11-21T03:41:34,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/c6a524c26efb456b91ede14a446ab1da, entries=150, sequenceid=16, filesize=11.7 K 2024-11-21T03:41:34,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/c535eadae0ee47e2be546c5c9f86a669 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/c535eadae0ee47e2be546c5c9f86a669 2024-11-21T03:41:34,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/c535eadae0ee47e2be546c5c9f86a669, entries=150, sequenceid=16, filesize=11.7 K 2024-11-21T03:41:34,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for f60eea9fbbad1295666bbf6cb6ce080c in 1718ms, sequenceid=16, compaction requested=false 2024-11-21T03:41:34,421 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-21T03:41:34,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:34,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T03:41:34,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:34,530 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing f60eea9fbbad1295666bbf6cb6ce080c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T03:41:34,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121535ff22b892f49d6a5683c3498742b59_f60eea9fbbad1295666bbf6cb6ce080c is 50, key is test_row_0/A:col10/1732160492735/Put/seqid=0 2024-11-21T03:41:34,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742187_1363 (size=12154) 2024-11-21T03:41:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T03:41:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:34,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:34,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160554920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:34,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:34,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:34,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:34,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160554921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:34,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160554922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:34,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160554923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:34,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:34,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160554926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:34,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:34,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:34,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:34,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:34,949 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121535ff22b892f49d6a5683c3498742b59_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121535ff22b892f49d6a5683c3498742b59_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:34,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/a803a1eca6dc4a1f91433e819bfe3b22, store: [table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:34,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/a803a1eca6dc4a1f91433e819bfe3b22 is 175, key is test_row_0/A:col10/1732160492735/Put/seqid=0 2024-11-21T03:41:34,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742188_1364 (size=30955) 2024-11-21T03:41:34,962 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/a803a1eca6dc4a1f91433e819bfe3b22 2024-11-21T03:41:34,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/34ef384b66204fbe9e53d29d097e48d4 is 50, key is test_row_0/B:col10/1732160492735/Put/seqid=0 2024-11-21T03:41:34,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742189_1365 (size=12001) 2024-11-21T03:41:35,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160555036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:35,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:35,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160555037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:35,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:35,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:35,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160555037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:35,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:35,042 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160555037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:35,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:35,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160555037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:35,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:35,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160555247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:35,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160555247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160555247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:35,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:35,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:35,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160555247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:35,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:35,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160555247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:35,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:35,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:35,372 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/34ef384b66204fbe9e53d29d097e48d4 2024-11-21T03:41:35,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/f05e2244b65e4240b00f3a86b1046835 is 50, key is test_row_0/C:col10/1732160492735/Put/seqid=0 2024-11-21T03:41:35,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742190_1366 (size=12001) 2024-11-21T03:41:35,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160555557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:35,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:35,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160555557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:35,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:35,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:35,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160555557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:35,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:35,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:35,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160555557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:35,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:35,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:35,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160555558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:35,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:35,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:35,782 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/f05e2244b65e4240b00f3a86b1046835 2024-11-21T03:41:35,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/a803a1eca6dc4a1f91433e819bfe3b22 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/a803a1eca6dc4a1f91433e819bfe3b22 2024-11-21T03:41:35,789 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/a803a1eca6dc4a1f91433e819bfe3b22, entries=150, sequenceid=40, filesize=30.2 K 2024-11-21T03:41:35,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/34ef384b66204fbe9e53d29d097e48d4 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/34ef384b66204fbe9e53d29d097e48d4 2024-11-21T03:41:35,793 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/34ef384b66204fbe9e53d29d097e48d4, entries=150, sequenceid=40, filesize=11.7 K 2024-11-21T03:41:35,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/f05e2244b65e4240b00f3a86b1046835 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/f05e2244b65e4240b00f3a86b1046835 2024-11-21T03:41:35,797 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/f05e2244b65e4240b00f3a86b1046835, entries=150, sequenceid=40, filesize=11.7 K 2024-11-21T03:41:35,797 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for f60eea9fbbad1295666bbf6cb6ce080c in 1267ms, sequenceid=40, compaction requested=false 2024-11-21T03:41:35,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:35,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:35,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-21T03:41:35,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-21T03:41:35,800 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-21T03:41:35,800 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1040 sec 2024-11-21T03:41:35,802 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 3.1150 sec 2024-11-21T03:41:36,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:36,071 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:41:36,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f60eea9fbbad1295666bbf6cb6ce080c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T03:41:36,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b35badc7e7de40a491cea95b72249d07_f60eea9fbbad1295666bbf6cb6ce080c is 50, key is test_row_0/A:col10/1732160496070/Put/seqid=0 2024-11-21T03:41:36,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742191_1367 (size=14594) 2024-11-21T03:41:36,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160556103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:36,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:36,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160556103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:36,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:36,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:36,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160556110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:36,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:36,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160556111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:36,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160556111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:36,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:36,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160556217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:36,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:36,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160556217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:36,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:36,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:36,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160556217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:36,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:36,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:36,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:36,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160556226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:36,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160556226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:36,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160556427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:36,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:36,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160556436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:36,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160556436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:36,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160556436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:36,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:36,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160556447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:36,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:36,484 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:36,487 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b35badc7e7de40a491cea95b72249d07_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b35badc7e7de40a491cea95b72249d07_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:36,488 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/2b3f6ae5239e4a07a9246a9dd508b212, store: [table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:36,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/2b3f6ae5239e4a07a9246a9dd508b212 is 175, key is test_row_0/A:col10/1732160496070/Put/seqid=0 2024-11-21T03:41:36,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742192_1368 (size=39549) 2024-11-21T03:41:36,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160556737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:36,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:36,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:36,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160556747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:36,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:36,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160556747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:36,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:36,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:36,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:36,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160556747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:36,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:36,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160556766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:36,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:36,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T03:41:36,857 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:41:36,858 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:41:36,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-21T03:41:36,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T03:41:36,860 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:41:36,860 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:41:36,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:41:36,898 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/2b3f6ae5239e4a07a9246a9dd508b212 2024-11-21T03:41:36,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/01f5634fdaa04f8fbcb875d631a00ebc is 50, key is test_row_0/B:col10/1732160496070/Put/seqid=0 2024-11-21T03:41:36,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742193_1369 (size=12001) 2024-11-21T03:41:36,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T03:41:37,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T03:41:37,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:37,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:37,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:37,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:37,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:37,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:37,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T03:41:37,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:37,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:37,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:37,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:37,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T03:41:37,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160557246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:37,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:37,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:37,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:37,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:37,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160557267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:37,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:37,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160557267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:37,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:37,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:37,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160557267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:37,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:37,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:37,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:37,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:37,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:37,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:37,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160557277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:37,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:37,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:37,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:37,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/01f5634fdaa04f8fbcb875d631a00ebc 2024-11-21T03:41:37,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T03:41:37,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:37,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:37,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:37,319 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:37,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:37,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/ca4c0d85548845789182fc7bc86260bd is 50, key is test_row_0/C:col10/1732160496070/Put/seqid=0 2024-11-21T03:41:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:37,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742194_1370 (size=12001) 2024-11-21T03:41:37,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T03:41:37,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:37,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:37,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:37,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:37,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:37,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:37,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T03:41:37,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T03:41:37,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:37,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:37,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:37,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:37,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:37,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:37,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/ca4c0d85548845789182fc7bc86260bd 2024-11-21T03:41:37,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/2b3f6ae5239e4a07a9246a9dd508b212 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/2b3f6ae5239e4a07a9246a9dd508b212 2024-11-21T03:41:37,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/2b3f6ae5239e4a07a9246a9dd508b212, entries=200, sequenceid=54, filesize=38.6 K 2024-11-21T03:41:37,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/01f5634fdaa04f8fbcb875d631a00ebc as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/01f5634fdaa04f8fbcb875d631a00ebc 2024-11-21T03:41:37,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/01f5634fdaa04f8fbcb875d631a00ebc, entries=150, sequenceid=54, filesize=11.7 K 2024-11-21T03:41:37,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/ca4c0d85548845789182fc7bc86260bd as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/ca4c0d85548845789182fc7bc86260bd 2024-11-21T03:41:37,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/ca4c0d85548845789182fc7bc86260bd, entries=150, sequenceid=54, filesize=11.7 K 2024-11-21T03:41:37,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for f60eea9fbbad1295666bbf6cb6ce080c in 1682ms, sequenceid=54, compaction requested=true 2024-11-21T03:41:37,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:37,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f60eea9fbbad1295666bbf6cb6ce080c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:41:37,753 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:37,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:37,753 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:37,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f60eea9fbbad1295666bbf6cb6ce080c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:41:37,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:37,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f60eea9fbbad1295666bbf6cb6ce080c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:41:37,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:37,754 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:37,754 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): f60eea9fbbad1295666bbf6cb6ce080c/B is initiating minor compaction (all files) 2024-11-21T03:41:37,754 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f60eea9fbbad1295666bbf6cb6ce080c/B in TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:37,755 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/c6a524c26efb456b91ede14a446ab1da, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/34ef384b66204fbe9e53d29d097e48d4, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/01f5634fdaa04f8fbcb875d631a00ebc] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp, totalSize=35.2 K 2024-11-21T03:41:37,755 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:37,755 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): f60eea9fbbad1295666bbf6cb6ce080c/A is initiating minor compaction (all files) 2024-11-21T03:41:37,755 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f60eea9fbbad1295666bbf6cb6ce080c/A in TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:37,755 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting c6a524c26efb456b91ede14a446ab1da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732160492700 2024-11-21T03:41:37,755 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/642b47f67c1548cc842addaa323cd83c, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/a803a1eca6dc4a1f91433e819bfe3b22, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/2b3f6ae5239e4a07a9246a9dd508b212] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp, totalSize=99.1 K 2024-11-21T03:41:37,755 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:37,755 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. files: [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/642b47f67c1548cc842addaa323cd83c, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/a803a1eca6dc4a1f91433e819bfe3b22, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/2b3f6ae5239e4a07a9246a9dd508b212] 2024-11-21T03:41:37,755 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 34ef384b66204fbe9e53d29d097e48d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732160492735 2024-11-21T03:41:37,755 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 642b47f67c1548cc842addaa323cd83c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732160492700 2024-11-21T03:41:37,755 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 01f5634fdaa04f8fbcb875d631a00ebc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732160494918 2024-11-21T03:41:37,756 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting a803a1eca6dc4a1f91433e819bfe3b22, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732160492735 2024-11-21T03:41:37,756 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b3f6ae5239e4a07a9246a9dd508b212, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732160494918 2024-11-21T03:41:37,760 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:37,761 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f60eea9fbbad1295666bbf6cb6ce080c#B#compaction#318 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:37,761 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/b6839741f1ad499196ae6da8938989d5 is 50, key is test_row_0/B:col10/1732160496070/Put/seqid=0 2024-11-21T03:41:37,762 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121d568d1dd0860492caf57ee9e078769a4_f60eea9fbbad1295666bbf6cb6ce080c store=[table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:37,765 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121d568d1dd0860492caf57ee9e078769a4_f60eea9fbbad1295666bbf6cb6ce080c, store=[table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:37,765 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d568d1dd0860492caf57ee9e078769a4_f60eea9fbbad1295666bbf6cb6ce080c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:37,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742195_1371 (size=12104) 2024-11-21T03:41:37,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742196_1372 (size=4469) 2024-11-21T03:41:37,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T03:41:37,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:37,779 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing f60eea9fbbad1295666bbf6cb6ce080c 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T03:41:37,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217bf98e31635d479eb624ecb79e8fc35d_f60eea9fbbad1295666bbf6cb6ce080c is 50, key is test_row_0/A:col10/1732160496099/Put/seqid=0 2024-11-21T03:41:37,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742197_1373 (size=12154) 2024-11-21T03:41:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T03:41:38,169 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f60eea9fbbad1295666bbf6cb6ce080c#A#compaction#319 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:38,169 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/8db21980d5544c7db9ddd2cc607b2648 is 175, key is test_row_0/A:col10/1732160496070/Put/seqid=0 2024-11-21T03:41:38,170 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/b6839741f1ad499196ae6da8938989d5 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/b6839741f1ad499196ae6da8938989d5 2024-11-21T03:41:38,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742198_1374 (size=31058) 2024-11-21T03:41:38,176 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f60eea9fbbad1295666bbf6cb6ce080c/B of f60eea9fbbad1295666bbf6cb6ce080c into b6839741f1ad499196ae6da8938989d5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:38,176 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:38,176 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., storeName=f60eea9fbbad1295666bbf6cb6ce080c/B, priority=13, startTime=1732160497753; duration=0sec 2024-11-21T03:41:38,176 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:38,176 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f60eea9fbbad1295666bbf6cb6ce080c:B 2024-11-21T03:41:38,176 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:38,177 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:38,177 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): f60eea9fbbad1295666bbf6cb6ce080c/C is initiating minor compaction (all files) 2024-11-21T03:41:38,177 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f60eea9fbbad1295666bbf6cb6ce080c/C in TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:38,177 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/c535eadae0ee47e2be546c5c9f86a669, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/f05e2244b65e4240b00f3a86b1046835, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/ca4c0d85548845789182fc7bc86260bd] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp, totalSize=35.2 K 2024-11-21T03:41:38,177 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting c535eadae0ee47e2be546c5c9f86a669, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732160492700 2024-11-21T03:41:38,178 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting f05e2244b65e4240b00f3a86b1046835, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732160492735 2024-11-21T03:41:38,178 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting ca4c0d85548845789182fc7bc86260bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732160494918 2024-11-21T03:41:38,184 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f60eea9fbbad1295666bbf6cb6ce080c#C#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:38,185 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/95896baa845b42e293f17ed84bbf6003 is 50, key is test_row_0/C:col10/1732160496070/Put/seqid=0 2024-11-21T03:41:38,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742199_1375 (size=12104) 2024-11-21T03:41:38,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:38,197 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217bf98e31635d479eb624ecb79e8fc35d_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217bf98e31635d479eb624ecb79e8fc35d_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:38,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/2833d89ed1a74c43a19e7f3809018273, store: [table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:38,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/2833d89ed1a74c43a19e7f3809018273 is 175, key is test_row_0/A:col10/1732160496099/Put/seqid=0 2024-11-21T03:41:38,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742200_1376 (size=30955) 2024-11-21T03:41:38,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:38,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:38,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:38,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:38,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160558297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:38,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160558298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:38,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:38,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160558299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:38,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160558314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:38,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160558315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:38,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:38,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160558417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:38,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160558417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:38,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160558417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:38,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160558426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:38,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:38,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160558437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:38,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:38,578 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/8db21980d5544c7db9ddd2cc607b2648 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/8db21980d5544c7db9ddd2cc607b2648 2024-11-21T03:41:38,582 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f60eea9fbbad1295666bbf6cb6ce080c/A of f60eea9fbbad1295666bbf6cb6ce080c into 8db21980d5544c7db9ddd2cc607b2648(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:38,582 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:38,582 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., storeName=f60eea9fbbad1295666bbf6cb6ce080c/A, priority=13, startTime=1732160497753; duration=0sec 2024-11-21T03:41:38,583 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:38,583 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f60eea9fbbad1295666bbf6cb6ce080c:A 2024-11-21T03:41:38,593 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/95896baa845b42e293f17ed84bbf6003 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/95896baa845b42e293f17ed84bbf6003 2024-11-21T03:41:38,597 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f60eea9fbbad1295666bbf6cb6ce080c/C of f60eea9fbbad1295666bbf6cb6ce080c into 95896baa845b42e293f17ed84bbf6003(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:38,597 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:38,597 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., storeName=f60eea9fbbad1295666bbf6cb6ce080c/C, priority=13, startTime=1732160497753; duration=0sec 2024-11-21T03:41:38,597 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:38,597 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f60eea9fbbad1295666bbf6cb6ce080c:C 2024-11-21T03:41:38,604 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/2833d89ed1a74c43a19e7f3809018273 2024-11-21T03:41:38,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/c2dfa996092341fe8edc56b0485af8f3 is 50, key is test_row_0/B:col10/1732160496099/Put/seqid=0 2024-11-21T03:41:38,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742201_1377 (size=12001) 2024-11-21T03:41:38,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:38,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160558636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:38,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:38,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,643 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:38,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160558636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:38,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:38,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:38,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160558636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:38,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:38,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:38,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160558636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:38,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:38,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:38,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160558647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:38,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:38,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:38,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160558947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:38,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:38,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160558947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:38,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:38,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:38,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:38,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160558947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:38,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:38,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160558947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:38,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:38,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:38,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:38,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160558966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:38,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T03:41:39,018 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/c2dfa996092341fe8edc56b0485af8f3 2024-11-21T03:41:39,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/91ed60e834454e8ea01edd0b7abcfd4d is 50, key is test_row_0/C:col10/1732160496099/Put/seqid=0 2024-11-21T03:41:39,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742202_1378 (size=12001) 2024-11-21T03:41:39,029 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/91ed60e834454e8ea01edd0b7abcfd4d 2024-11-21T03:41:39,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/2833d89ed1a74c43a19e7f3809018273 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/2833d89ed1a74c43a19e7f3809018273 2024-11-21T03:41:39,035 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/2833d89ed1a74c43a19e7f3809018273, entries=150, sequenceid=77, filesize=30.2 K 2024-11-21T03:41:39,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/c2dfa996092341fe8edc56b0485af8f3 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/c2dfa996092341fe8edc56b0485af8f3 2024-11-21T03:41:39,039 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/c2dfa996092341fe8edc56b0485af8f3, entries=150, sequenceid=77, filesize=11.7 K 2024-11-21T03:41:39,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/91ed60e834454e8ea01edd0b7abcfd4d as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/91ed60e834454e8ea01edd0b7abcfd4d 2024-11-21T03:41:39,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,043 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/91ed60e834454e8ea01edd0b7abcfd4d, entries=150, sequenceid=77, filesize=11.7 K 2024-11-21T03:41:39,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,044 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for f60eea9fbbad1295666bbf6cb6ce080c in 1265ms, sequenceid=77, compaction requested=false 2024-11-21T03:41:39,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:39,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:39,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-21T03:41:39,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-21T03:41:39,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-21T03:41:39,047 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1850 sec 2024-11-21T03:41:39,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,048 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.1890 sec 2024-11-21T03:41:39,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:39,472 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:41:39,472 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f60eea9fbbad1295666bbf6cb6ce080c 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T03:41:39,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216e787c43b65c4cc69c464d2989e48ce6_f60eea9fbbad1295666bbf6cb6ce080c is 50, key is test_row_0/A:col10/1732160498311/Put/seqid=0 2024-11-21T03:41:39,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742203_1379 (size=12154) 2024-11-21T03:41:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,489 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,494 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216e787c43b65c4cc69c464d2989e48ce6_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216e787c43b65c4cc69c464d2989e48ce6_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:39,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,494 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/a919f5770f98460d9f83adc6408c7c6d, store: [table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:39,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/a919f5770f98460d9f83adc6408c7c6d is 175, key is test_row_0/A:col10/1732160498311/Put/seqid=0 2024-11-21T03:41:39,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742204_1380 (size=30951) 2024-11-21T03:41:39,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:39,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:39,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160559559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:39,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:39,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160559561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:39,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:39,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:39,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160559562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:39,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160559563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:39,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160559563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:39,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:39,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,575 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:39,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:39,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160559677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:39,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:39,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:39,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160559677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:39,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:39,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:39,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160559677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:39,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:39,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:39,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160559677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:39,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:39,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:39,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160559677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:39,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:39,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160559887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:39,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160559887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:39,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:39,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:39,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160559887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:39,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:39,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:39,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160559887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:39,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:39,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160559887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:39,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:39,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:39,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:39,908 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/a919f5770f98460d9f83adc6408c7c6d 2024-11-21T03:41:39,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/550f473b5836488cbc3934739b6e2381 is 50, key is test_row_0/B:col10/1732160498311/Put/seqid=0 2024-11-21T03:41:39,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742205_1381 (size=9657) 2024-11-21T03:41:40,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160560196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:40,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160560197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160560197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:40,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160560197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:40,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160560207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:40,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:40,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/550f473b5836488cbc3934739b6e2381 2024-11-21T03:41:40,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/cb82fc47a77f44bbab1f9c14ceda70e5 is 50, key is test_row_0/C:col10/1732160498311/Put/seqid=0 2024-11-21T03:41:40,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742206_1382 (size=9657) 2024-11-21T03:41:40,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/cb82fc47a77f44bbab1f9c14ceda70e5 2024-11-21T03:41:40,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/a919f5770f98460d9f83adc6408c7c6d as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/a919f5770f98460d9f83adc6408c7c6d 2024-11-21T03:41:40,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/a919f5770f98460d9f83adc6408c7c6d, entries=150, sequenceid=95, filesize=30.2 K 2024-11-21T03:41:40,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/550f473b5836488cbc3934739b6e2381 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/550f473b5836488cbc3934739b6e2381 2024-11-21T03:41:40,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/550f473b5836488cbc3934739b6e2381, entries=100, sequenceid=95, filesize=9.4 K 2024-11-21T03:41:40,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/cb82fc47a77f44bbab1f9c14ceda70e5 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/cb82fc47a77f44bbab1f9c14ceda70e5 2024-11-21T03:41:40,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/cb82fc47a77f44bbab1f9c14ceda70e5, entries=100, sequenceid=95, filesize=9.4 K 2024-11-21T03:41:40,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for f60eea9fbbad1295666bbf6cb6ce080c in 874ms, sequenceid=95, compaction requested=true 2024-11-21T03:41:40,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:40,346 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:40,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f60eea9fbbad1295666bbf6cb6ce080c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:41:40,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:40,346 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:40,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f60eea9fbbad1295666bbf6cb6ce080c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:41:40,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:40,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f60eea9fbbad1295666bbf6cb6ce080c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:41:40,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:40,347 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92964 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:40,347 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): f60eea9fbbad1295666bbf6cb6ce080c/A is initiating minor compaction (all files) 2024-11-21T03:41:40,347 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f60eea9fbbad1295666bbf6cb6ce080c/A in TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:40,348 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/8db21980d5544c7db9ddd2cc607b2648, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/2833d89ed1a74c43a19e7f3809018273, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/a919f5770f98460d9f83adc6408c7c6d] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp, totalSize=90.8 K 2024-11-21T03:41:40,348 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:40,348 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. files: [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/8db21980d5544c7db9ddd2cc607b2648, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/2833d89ed1a74c43a19e7f3809018273, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/a919f5770f98460d9f83adc6408c7c6d] 2024-11-21T03:41:40,348 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:40,348 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): f60eea9fbbad1295666bbf6cb6ce080c/B is initiating minor compaction (all files) 2024-11-21T03:41:40,348 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f60eea9fbbad1295666bbf6cb6ce080c/B in TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:40,348 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/b6839741f1ad499196ae6da8938989d5, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/c2dfa996092341fe8edc56b0485af8f3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/550f473b5836488cbc3934739b6e2381] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp, totalSize=33.0 K 2024-11-21T03:41:40,348 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8db21980d5544c7db9ddd2cc607b2648, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732160494918 2024-11-21T03:41:40,350 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting b6839741f1ad499196ae6da8938989d5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732160494918 2024-11-21T03:41:40,350 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2833d89ed1a74c43a19e7f3809018273, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732160496099 2024-11-21T03:41:40,350 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting c2dfa996092341fe8edc56b0485af8f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732160496099 2024-11-21T03:41:40,350 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting a919f5770f98460d9f83adc6408c7c6d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732160498311 2024-11-21T03:41:40,350 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 550f473b5836488cbc3934739b6e2381, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732160498311 2024-11-21T03:41:40,356 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f60eea9fbbad1295666bbf6cb6ce080c#B#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:40,357 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/8e43e5069773422284159a183b01e545 is 50, key is test_row_0/B:col10/1732160498311/Put/seqid=0 2024-11-21T03:41:40,365 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:40,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742207_1383 (size=12207) 2024-11-21T03:41:40,369 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121f660603b230141a69774c439c511507a_f60eea9fbbad1295666bbf6cb6ce080c store=[table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:40,371 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121f660603b230141a69774c439c511507a_f60eea9fbbad1295666bbf6cb6ce080c, store=[table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:40,371 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f660603b230141a69774c439c511507a_f60eea9fbbad1295666bbf6cb6ce080c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:40,375 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/8e43e5069773422284159a183b01e545 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/8e43e5069773422284159a183b01e545 2024-11-21T03:41:40,379 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f60eea9fbbad1295666bbf6cb6ce080c/B of f60eea9fbbad1295666bbf6cb6ce080c into 8e43e5069773422284159a183b01e545(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:40,379 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:40,379 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., storeName=f60eea9fbbad1295666bbf6cb6ce080c/B, priority=13, startTime=1732160500346; duration=0sec 2024-11-21T03:41:40,379 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:40,380 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f60eea9fbbad1295666bbf6cb6ce080c:B 2024-11-21T03:41:40,380 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:40,380 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:40,381 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): f60eea9fbbad1295666bbf6cb6ce080c/C is initiating minor compaction (all files) 2024-11-21T03:41:40,381 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f60eea9fbbad1295666bbf6cb6ce080c/C in TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:40,381 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/95896baa845b42e293f17ed84bbf6003, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/91ed60e834454e8ea01edd0b7abcfd4d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/cb82fc47a77f44bbab1f9c14ceda70e5] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp, totalSize=33.0 K 2024-11-21T03:41:40,383 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 95896baa845b42e293f17ed84bbf6003, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732160494918 2024-11-21T03:41:40,383 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 91ed60e834454e8ea01edd0b7abcfd4d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732160496099 2024-11-21T03:41:40,383 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting cb82fc47a77f44bbab1f9c14ceda70e5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732160498311 2024-11-21T03:41:40,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742208_1384 (size=4469) 2024-11-21T03:41:40,394 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f60eea9fbbad1295666bbf6cb6ce080c#C#compaction#329 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:40,395 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/133fff001a964351aeb246115fa3d4a7 is 50, key is test_row_0/C:col10/1732160498311/Put/seqid=0 2024-11-21T03:41:40,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742209_1385 (size=12207) 2024-11-21T03:41:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:40,711 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:41:40,711 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f60eea9fbbad1295666bbf6cb6ce080c 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-21T03:41:40,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411218b6a7f5b1c9c49df93bc6c0891eb859b_f60eea9fbbad1295666bbf6cb6ce080c is 50, key is test_row_0/A:col10/1732160500710/Put/seqid=0 2024-11-21T03:41:40,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742210_1386 (size=17034) 2024-11-21T03:41:40,722 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:40,725 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411218b6a7f5b1c9c49df93bc6c0891eb859b_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218b6a7f5b1c9c49df93bc6c0891eb859b_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:40,725 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/e1085fbe226a49619a12d592384da3d9, store: [table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:40,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/e1085fbe226a49619a12d592384da3d9 is 175, key is test_row_0/A:col10/1732160500710/Put/seqid=0 2024-11-21T03:41:40,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742211_1387 (size=48139) 2024-11-21T03:41:40,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:40,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160560726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:40,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:40,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:40,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160560727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:40,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:40,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:40,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160560728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:40,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:40,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160560732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160560733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:40,791 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f60eea9fbbad1295666bbf6cb6ce080c#A#compaction#328 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:40,792 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/013e90b5d81e454e816a3292014fda3c is 175, key is test_row_0/A:col10/1732160498311/Put/seqid=0 2024-11-21T03:41:40,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742212_1388 (size=31268) 2024-11-21T03:41:40,803 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/133fff001a964351aeb246115fa3d4a7 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/133fff001a964351aeb246115fa3d4a7 2024-11-21T03:41:40,807 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f60eea9fbbad1295666bbf6cb6ce080c/C of f60eea9fbbad1295666bbf6cb6ce080c into 133fff001a964351aeb246115fa3d4a7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:40,807 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:40,807 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., storeName=f60eea9fbbad1295666bbf6cb6ce080c/C, priority=13, startTime=1732160500346; duration=0sec 2024-11-21T03:41:40,808 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:40,808 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f60eea9fbbad1295666bbf6cb6ce080c:C 2024-11-21T03:41:40,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:40,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160560837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:40,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:40,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:40,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160560846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:40,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160560846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:40,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:40,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:40,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160560846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:40,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:40,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160560847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:40,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:40,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:40,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:41,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T03:41:41,017 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:41:41,018 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:41:41,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-21T03:41:41,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T03:41:41,019 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:41:41,019 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:41:41,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:41:41,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:41,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160561046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:41,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:41,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160561057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:41,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:41,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160561057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:41,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:41,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:41,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160561057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:41,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:41,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:41,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:41,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160561057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:41,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:41,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T03:41:41,131 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/e1085fbe226a49619a12d592384da3d9 2024-11-21T03:41:41,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/81d6621bf590477793faa83040f31194 is 50, key is test_row_0/B:col10/1732160500710/Put/seqid=0 2024-11-21T03:41:41,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742213_1389 (size=12001) 2024-11-21T03:41:41,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-21T03:41:41,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:41,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:41,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:41,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:41,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:41,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:41,199 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/013e90b5d81e454e816a3292014fda3c as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/013e90b5d81e454e816a3292014fda3c 2024-11-21T03:41:41,203 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f60eea9fbbad1295666bbf6cb6ce080c/A of f60eea9fbbad1295666bbf6cb6ce080c into 013e90b5d81e454e816a3292014fda3c(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:41,203 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:41,203 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., storeName=f60eea9fbbad1295666bbf6cb6ce080c/A, priority=13, startTime=1732160500346; duration=0sec 2024-11-21T03:41:41,204 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:41,204 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f60eea9fbbad1295666bbf6cb6ce080c:A 2024-11-21T03:41:41,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-21T03:41:41,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:41,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:41,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:41,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:41,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:41,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:41,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T03:41:41,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160561356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:41,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:41,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160561366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:41,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:41,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160561367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:41,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,371 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:41,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160561367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:41,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:41,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160561367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:41,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:41,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-21T03:41:41,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:41,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:41,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:41,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:41,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:41,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:41,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/81d6621bf590477793faa83040f31194 2024-11-21T03:41:41,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/8930bdbb178540bd8fb10d5ea49d70fe is 50, key is test_row_0/C:col10/1732160500710/Put/seqid=0 2024-11-21T03:41:41,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742214_1390 (size=12001) 2024-11-21T03:41:41,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-21T03:41:41,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:41,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:41,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:41,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:41,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:41,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:41,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T03:41:41,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-21T03:41:41,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:41,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:41,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:41,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:41,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:41,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:41,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:41,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160561866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:41,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:41,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:41,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160561877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:41,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:41,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:41,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160561877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:41,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:41,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:41,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160561877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:41,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:41,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:41,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160561877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:41,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:41,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:41,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-21T03:41:41,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:41,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:41,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:41,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:41,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:41,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/8930bdbb178540bd8fb10d5ea49d70fe 2024-11-21T03:41:41,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/e1085fbe226a49619a12d592384da3d9 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/e1085fbe226a49619a12d592384da3d9 2024-11-21T03:41:41,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/e1085fbe226a49619a12d592384da3d9, entries=250, sequenceid=119, filesize=47.0 K 2024-11-21T03:41:41,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/81d6621bf590477793faa83040f31194 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/81d6621bf590477793faa83040f31194 2024-11-21T03:41:41,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/81d6621bf590477793faa83040f31194, entries=150, sequenceid=119, filesize=11.7 K 2024-11-21T03:41:41,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/8930bdbb178540bd8fb10d5ea49d70fe as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/8930bdbb178540bd8fb10d5ea49d70fe 2024-11-21T03:41:41,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/8930bdbb178540bd8fb10d5ea49d70fe, entries=150, sequenceid=119, filesize=11.7 K 2024-11-21T03:41:41,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for f60eea9fbbad1295666bbf6cb6ce080c in 1254ms, sequenceid=119, compaction requested=false 2024-11-21T03:41:41,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:42,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-21T03:41:42,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:42,087 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing f60eea9fbbad1295666bbf6cb6ce080c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T03:41:42,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bd0fa9b3fd4b436a97dc5b2251cacbd9_f60eea9fbbad1295666bbf6cb6ce080c is 50, key is test_row_0/A:col10/1732160500730/Put/seqid=0 2024-11-21T03:41:42,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742215_1391 (size=12304) 2024-11-21T03:41:42,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T03:41:42,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:42,537 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bd0fa9b3fd4b436a97dc5b2251cacbd9_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bd0fa9b3fd4b436a97dc5b2251cacbd9_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:42,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/995adb9eb189485eae96e654801c3a81, store: [table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:42,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/995adb9eb189485eae96e654801c3a81 is 175, key is test_row_0/A:col10/1732160500730/Put/seqid=0 2024-11-21T03:41:42,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742216_1392 (size=31105) 2024-11-21T03:41:42,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:42,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:42,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160562924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:42,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:42,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:42,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160562925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:42,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:42,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:42,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:42,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:42,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:42,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160562926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:42,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:42,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:42,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:42,942 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/995adb9eb189485eae96e654801c3a81 2024-11-21T03:41:42,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:42,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160562933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:42,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:42,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160562934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:42,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:42,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/48899da4bcaa4cc5849cffeb0459845b is 50, key is test_row_0/B:col10/1732160500730/Put/seqid=0 2024-11-21T03:41:42,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742217_1393 (size=12151) 2024-11-21T03:41:43,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:43,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160563036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:43,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:43,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160563037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:43,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:43,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:43,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160563037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:43,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:43,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:43,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:43,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160563047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:43,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160563047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:43,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T03:41:43,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:43,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160563247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:43,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:43,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160563247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:43,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160563247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:43,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:43,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160563256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:43,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:43,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:43,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160563266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:43,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:43,354 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/48899da4bcaa4cc5849cffeb0459845b 2024-11-21T03:41:43,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/d60f7aa07ca44880b00f619a7af7f82e is 50, key is test_row_0/C:col10/1732160500730/Put/seqid=0 2024-11-21T03:41:43,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742218_1394 (size=12151) 2024-11-21T03:41:43,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:43,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160563556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:43,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:43,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:43,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160563567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:43,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:43,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160563567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:43,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:43,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:43,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160563567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:43,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:43,572 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:43,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:43,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160563577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:43,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:43,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:43,768 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/d60f7aa07ca44880b00f619a7af7f82e 2024-11-21T03:41:43,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/995adb9eb189485eae96e654801c3a81 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/995adb9eb189485eae96e654801c3a81 2024-11-21T03:41:43,774 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/995adb9eb189485eae96e654801c3a81, entries=150, sequenceid=134, filesize=30.4 K 2024-11-21T03:41:43,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/48899da4bcaa4cc5849cffeb0459845b as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/48899da4bcaa4cc5849cffeb0459845b 2024-11-21T03:41:43,777 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/48899da4bcaa4cc5849cffeb0459845b, entries=150, sequenceid=134, filesize=11.9 K 2024-11-21T03:41:43,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/d60f7aa07ca44880b00f619a7af7f82e as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/d60f7aa07ca44880b00f619a7af7f82e 2024-11-21T03:41:43,781 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/d60f7aa07ca44880b00f619a7af7f82e, entries=150, sequenceid=134, filesize=11.9 K 2024-11-21T03:41:43,782 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for f60eea9fbbad1295666bbf6cb6ce080c in 1695ms, sequenceid=134, compaction requested=true 2024-11-21T03:41:43,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:43,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:43,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-21T03:41:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-21T03:41:43,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-21T03:41:43,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7640 sec 2024-11-21T03:41:43,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.7670 sec 2024-11-21T03:41:44,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:44,072 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:41:44,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f60eea9fbbad1295666bbf6cb6ce080c 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T03:41:44,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c56a341ec30e4da6a902b0472940e3a8_f60eea9fbbad1295666bbf6cb6ce080c is 50, key is test_row_0/A:col10/1732160504071/Put/seqid=0 2024-11-21T03:41:44,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742219_1395 (size=14794) 2024-11-21T03:41:44,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160564097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:44,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160564097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:44,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:44,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160564098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:44,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:44,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:44,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:44,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160564099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:44,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:44,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:44,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160564099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:44,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:44,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:44,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160564207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:44,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:44,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160564217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:44,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160564217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160564217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:44,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:44,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160564217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:44,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:44,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:44,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160564417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:44,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:44,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160564437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:44,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160564437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:44,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160564437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:44,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160564437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:44,485 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:44,487 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c56a341ec30e4da6a902b0472940e3a8_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c56a341ec30e4da6a902b0472940e3a8_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:44,488 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/0a400ce1976d455482b9d1c5ee5711e5, store: [table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:44,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/0a400ce1976d455482b9d1c5ee5711e5 is 175, key is test_row_0/A:col10/1732160504071/Put/seqid=0 2024-11-21T03:41:44,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742220_1396 (size=39749) 2024-11-21T03:41:44,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:44,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160564726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:44,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:44,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:44,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160564747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:44,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:44,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160564747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:44,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:44,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:44,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160564757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:44,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:44,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160564757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:44,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,763 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:44,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:44,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:44,894 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/0a400ce1976d455482b9d1c5ee5711e5 2024-11-21T03:41:44,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/149c1dbfbf5b49c382116a3aa9869982 is 50, key is test_row_0/B:col10/1732160504071/Put/seqid=0 2024-11-21T03:41:44,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742221_1397 (size=12151) 2024-11-21T03:41:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T03:41:45,177 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:41:45,178 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:41:45,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-21T03:41:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T03:41:45,179 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:41:45,180 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:41:45,180 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:41:45,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:45,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160565247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:45,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:45,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:45,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:45,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:45,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160565257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:45,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:45,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:45,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:45,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:45,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160565267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:45,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:45,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160565267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:45,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:45,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160565276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:45,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:45,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:45,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:45,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T03:41:45,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/149c1dbfbf5b49c382116a3aa9869982 2024-11-21T03:41:45,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/ef63fa31fbd54a7d8be8097c7b4b4143 is 50, key is test_row_0/C:col10/1732160504071/Put/seqid=0 2024-11-21T03:41:45,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742222_1398 (size=12151) 2024-11-21T03:41:45,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/ef63fa31fbd54a7d8be8097c7b4b4143 2024-11-21T03:41:45,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/0a400ce1976d455482b9d1c5ee5711e5 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/0a400ce1976d455482b9d1c5ee5711e5 2024-11-21T03:41:45,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/0a400ce1976d455482b9d1c5ee5711e5, entries=200, sequenceid=158, filesize=38.8 K 2024-11-21T03:41:45,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/149c1dbfbf5b49c382116a3aa9869982 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/149c1dbfbf5b49c382116a3aa9869982 2024-11-21T03:41:45,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/149c1dbfbf5b49c382116a3aa9869982, entries=150, sequenceid=158, filesize=11.9 K 2024-11-21T03:41:45,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/ef63fa31fbd54a7d8be8097c7b4b4143 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/ef63fa31fbd54a7d8be8097c7b4b4143 2024-11-21T03:41:45,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T03:41:45,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:45,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:45,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:45,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:45,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:45,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/ef63fa31fbd54a7d8be8097c7b4b4143, entries=150, sequenceid=158, filesize=11.9 K 2024-11-21T03:41:45,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for f60eea9fbbad1295666bbf6cb6ce080c in 1266ms, sequenceid=158, compaction requested=true 2024-11-21T03:41:45,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:45,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f60eea9fbbad1295666bbf6cb6ce080c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:41:45,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:45,338 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:41:45,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f60eea9fbbad1295666bbf6cb6ce080c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:41:45,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:45,338 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:41:45,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f60eea9fbbad1295666bbf6cb6ce080c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:41:45,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:45,339 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150261 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:41:45,339 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:41:45,339 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): f60eea9fbbad1295666bbf6cb6ce080c/A is initiating minor compaction (all files) 2024-11-21T03:41:45,339 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): f60eea9fbbad1295666bbf6cb6ce080c/B is initiating minor compaction (all files) 2024-11-21T03:41:45,339 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f60eea9fbbad1295666bbf6cb6ce080c/A in TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:45,339 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f60eea9fbbad1295666bbf6cb6ce080c/B in TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:45,339 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/013e90b5d81e454e816a3292014fda3c, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/e1085fbe226a49619a12d592384da3d9, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/995adb9eb189485eae96e654801c3a81, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/0a400ce1976d455482b9d1c5ee5711e5] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp, totalSize=146.7 K 2024-11-21T03:41:45,339 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/8e43e5069773422284159a183b01e545, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/81d6621bf590477793faa83040f31194, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/48899da4bcaa4cc5849cffeb0459845b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/149c1dbfbf5b49c382116a3aa9869982] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp, totalSize=47.4 K 2024-11-21T03:41:45,339 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:45,340 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. files: [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/013e90b5d81e454e816a3292014fda3c, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/e1085fbe226a49619a12d592384da3d9, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/995adb9eb189485eae96e654801c3a81, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/0a400ce1976d455482b9d1c5ee5711e5] 2024-11-21T03:41:45,341 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e43e5069773422284159a183b01e545, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732160496110 2024-11-21T03:41:45,341 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 013e90b5d81e454e816a3292014fda3c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732160496110 2024-11-21T03:41:45,342 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 81d6621bf590477793faa83040f31194, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732160499559 2024-11-21T03:41:45,342 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1085fbe226a49619a12d592384da3d9, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732160499543 2024-11-21T03:41:45,342 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 48899da4bcaa4cc5849cffeb0459845b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732160500727 2024-11-21T03:41:45,342 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 149c1dbfbf5b49c382116a3aa9869982, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732160502931 2024-11-21T03:41:45,342 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 995adb9eb189485eae96e654801c3a81, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732160500727 2024-11-21T03:41:45,342 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a400ce1976d455482b9d1c5ee5711e5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732160502913 2024-11-21T03:41:45,347 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:45,350 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f60eea9fbbad1295666bbf6cb6ce080c#B#compaction#339 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:45,350 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411217fc09f13f0b04d4fa45d1446caf35245_f60eea9fbbad1295666bbf6cb6ce080c store=[table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:45,350 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/4863f19a55ad4e5a9068ca1712207bd0 is 50, key is test_row_0/B:col10/1732160504071/Put/seqid=0 2024-11-21T03:41:45,353 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411217fc09f13f0b04d4fa45d1446caf35245_f60eea9fbbad1295666bbf6cb6ce080c, store=[table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:45,353 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217fc09f13f0b04d4fa45d1446caf35245_f60eea9fbbad1295666bbf6cb6ce080c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:45,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742223_1399 (size=12493) 2024-11-21T03:41:45,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742224_1400 (size=4469) 2024-11-21T03:41:45,374 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f60eea9fbbad1295666bbf6cb6ce080c#A#compaction#340 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:45,375 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/de2297d7197e4974a0aa4aca3ec5c793 is 175, key is test_row_0/A:col10/1732160504071/Put/seqid=0 2024-11-21T03:41:45,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742225_1401 (size=31447) 2024-11-21T03:41:45,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T03:41:45,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:45,485 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing f60eea9fbbad1295666bbf6cb6ce080c 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T03:41:45,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a4339b3a6250440fbd58c8dd301a42b7_f60eea9fbbad1295666bbf6cb6ce080c is 50, key is test_row_0/A:col10/1732160504097/Put/seqid=0 2024-11-21T03:41:45,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742226_1402 (size=12304) 2024-11-21T03:41:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T03:41:45,768 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/4863f19a55ad4e5a9068ca1712207bd0 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/4863f19a55ad4e5a9068ca1712207bd0 2024-11-21T03:41:45,772 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f60eea9fbbad1295666bbf6cb6ce080c/B of f60eea9fbbad1295666bbf6cb6ce080c into 4863f19a55ad4e5a9068ca1712207bd0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:45,772 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:45,772 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., storeName=f60eea9fbbad1295666bbf6cb6ce080c/B, priority=12, startTime=1732160505338; duration=0sec 2024-11-21T03:41:45,772 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:45,772 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f60eea9fbbad1295666bbf6cb6ce080c:B 2024-11-21T03:41:45,772 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:41:45,773 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:41:45,773 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): f60eea9fbbad1295666bbf6cb6ce080c/C is initiating minor compaction (all files) 2024-11-21T03:41:45,773 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f60eea9fbbad1295666bbf6cb6ce080c/C in TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:45,773 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/133fff001a964351aeb246115fa3d4a7, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/8930bdbb178540bd8fb10d5ea49d70fe, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/d60f7aa07ca44880b00f619a7af7f82e, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/ef63fa31fbd54a7d8be8097c7b4b4143] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp, totalSize=47.4 K 2024-11-21T03:41:45,773 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 133fff001a964351aeb246115fa3d4a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732160496110 2024-11-21T03:41:45,773 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 8930bdbb178540bd8fb10d5ea49d70fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732160499559 2024-11-21T03:41:45,774 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting d60f7aa07ca44880b00f619a7af7f82e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732160500727 2024-11-21T03:41:45,774 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting ef63fa31fbd54a7d8be8097c7b4b4143, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732160502931 2024-11-21T03:41:45,782 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f60eea9fbbad1295666bbf6cb6ce080c#C#compaction#342 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:45,783 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/ceafe454e7654a4b98faae8d5dfce2c3 is 50, key is test_row_0/C:col10/1732160504071/Put/seqid=0 2024-11-21T03:41:45,786 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/de2297d7197e4974a0aa4aca3ec5c793 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/de2297d7197e4974a0aa4aca3ec5c793 2024-11-21T03:41:45,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742227_1403 (size=12493) 2024-11-21T03:41:45,791 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f60eea9fbbad1295666bbf6cb6ce080c/A of f60eea9fbbad1295666bbf6cb6ce080c into de2297d7197e4974a0aa4aca3ec5c793(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:45,791 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:45,791 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., storeName=f60eea9fbbad1295666bbf6cb6ce080c/A, priority=12, startTime=1732160505338; duration=0sec 2024-11-21T03:41:45,791 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:45,791 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f60eea9fbbad1295666bbf6cb6ce080c:A 2024-11-21T03:41:45,792 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/ceafe454e7654a4b98faae8d5dfce2c3 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/ceafe454e7654a4b98faae8d5dfce2c3 2024-11-21T03:41:45,796 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f60eea9fbbad1295666bbf6cb6ce080c/C of f60eea9fbbad1295666bbf6cb6ce080c into ceafe454e7654a4b98faae8d5dfce2c3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:45,796 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:45,797 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., storeName=f60eea9fbbad1295666bbf6cb6ce080c/C, priority=12, startTime=1732160505338; duration=0sec 2024-11-21T03:41:45,797 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:45,797 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f60eea9fbbad1295666bbf6cb6ce080c:C 2024-11-21T03:41:45,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T03:41:45,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:45,899 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a4339b3a6250440fbd58c8dd301a42b7_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a4339b3a6250440fbd58c8dd301a42b7_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:45,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/93b45344029c4b768de664e1399de782, store: [table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:45,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/93b45344029c4b768de664e1399de782 is 175, key is test_row_0/A:col10/1732160504097/Put/seqid=0 2024-11-21T03:41:45,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742228_1404 (size=31105) 2024-11-21T03:41:46,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:46,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:46,307 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/93b45344029c4b768de664e1399de782 2024-11-21T03:41:46,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/7e2fbdaa820f4a0f84218b71f24471e5 is 50, key is test_row_0/B:col10/1732160504097/Put/seqid=0 2024-11-21T03:41:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T03:41:46,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742229_1405 (size=12151) 2024-11-21T03:41:46,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:46,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160566309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:46,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160566317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:46,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160566318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:46,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160566319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:46,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160566319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:46,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160566427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:46,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160566427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:46,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160566436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160566437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160566437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:46,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:46,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160566636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:46,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:46,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:46,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160566646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:46,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:46,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160566647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:46,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:46,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:46,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160566647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:46,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:46,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:46,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:46,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160566647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:46,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:46,718 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/7e2fbdaa820f4a0f84218b71f24471e5 2024-11-21T03:41:46,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/e2fc5efd5ac04d98a3a83095f7bbe65d is 50, key is test_row_0/C:col10/1732160504097/Put/seqid=0 2024-11-21T03:41:46,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742230_1406 (size=12151) 2024-11-21T03:41:46,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:46,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160566947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:46,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:46,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160566956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:46,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:46,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160566956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:46,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:46,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160566956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:46,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:46,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:46,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160566967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:46,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:46,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:47,127 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/e2fc5efd5ac04d98a3a83095f7bbe65d 2024-11-21T03:41:47,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/93b45344029c4b768de664e1399de782 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/93b45344029c4b768de664e1399de782 2024-11-21T03:41:47,133 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/93b45344029c4b768de664e1399de782, entries=150, sequenceid=171, filesize=30.4 K 2024-11-21T03:41:47,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/7e2fbdaa820f4a0f84218b71f24471e5 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/7e2fbdaa820f4a0f84218b71f24471e5 2024-11-21T03:41:47,137 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/7e2fbdaa820f4a0f84218b71f24471e5, entries=150, sequenceid=171, filesize=11.9 K 2024-11-21T03:41:47,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/e2fc5efd5ac04d98a3a83095f7bbe65d as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/e2fc5efd5ac04d98a3a83095f7bbe65d 2024-11-21T03:41:47,140 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/e2fc5efd5ac04d98a3a83095f7bbe65d, entries=150, sequenceid=171, filesize=11.9 K 2024-11-21T03:41:47,141 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for f60eea9fbbad1295666bbf6cb6ce080c in 1656ms, sequenceid=171, compaction requested=false 2024-11-21T03:41:47,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:47,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:47,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-21T03:41:47,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-21T03:41:47,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-21T03:41:47,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9620 sec 2024-11-21T03:41:47,145 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.9660 sec 2024-11-21T03:41:47,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T03:41:47,326 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:41:47,327 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:41:47,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-21T03:41:47,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T03:41:47,328 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:41:47,329 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:41:47,329 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:41:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T03:41:47,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:47,461 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:41:47,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f60eea9fbbad1295666bbf6cb6ce080c 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T03:41:47,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411218fb9d454c1904a37a6745c6fe3d1f9b1_f60eea9fbbad1295666bbf6cb6ce080c is 50, key is test_row_0/A:col10/1732160506297/Put/seqid=0 2024-11-21T03:41:47,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742231_1407 (size=14794) 2024-11-21T03:41:47,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T03:41:47,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:47,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:47,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:47,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:47,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:47,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160567477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:47,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:47,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160567481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:47,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160567482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:47,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:47,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160567484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:47,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:47,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:47,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:47,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160567487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:47,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160567587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:47,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:47,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:47,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:47,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160567597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:47,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:47,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:47,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160567597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:47,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:47,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:47,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:47,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:47,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160567597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:47,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:47,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:47,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:47,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T03:41:47,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:47,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:47,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:47,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:47,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T03:41:47,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T03:41:47,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:47,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:47,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:47,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:47,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:47,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160567797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:47,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:47,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:47,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:47,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160567807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:47,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:47,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:47,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160567807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:47,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:47,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:47,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:47,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:47,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160567816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:47,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:47,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:47,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:47,875 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:47,878 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411218fb9d454c1904a37a6745c6fe3d1f9b1_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218fb9d454c1904a37a6745c6fe3d1f9b1_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:47,878 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/d6192e3973144862b330620cf00cd35e, store: [table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:47,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/d6192e3973144862b330620cf00cd35e is 175, key is test_row_0/A:col10/1732160506297/Put/seqid=0 2024-11-21T03:41:47,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742232_1408 (size=39749) 2024-11-21T03:41:47,885 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/d6192e3973144862b330620cf00cd35e 2024-11-21T03:41:47,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/7ca7764c19d94f449e4f1a7c502945e4 is 50, key is test_row_0/B:col10/1732160506297/Put/seqid=0 2024-11-21T03:41:47,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742233_1409 (size=12151) 2024-11-21T03:41:47,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/7ca7764c19d94f449e4f1a7c502945e4 2024-11-21T03:41:47,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/2638e58e7ed04a0c8d4da7edf6df3f6d is 50, key is test_row_0/C:col10/1732160506297/Put/seqid=0 2024-11-21T03:41:47,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742234_1410 (size=12151) 2024-11-21T03:41:47,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T03:41:47,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:47,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:47,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:47,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:47,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T03:41:48,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T03:41:48,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:48,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:48,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:48,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:48,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:48,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:48,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:48,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160568107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:48,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:48,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:48,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160568117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:48,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160568117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:48,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:48,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160568126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:48,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:48,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T03:41:48,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:48,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:48,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:48,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:48,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:48,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:48,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/2638e58e7ed04a0c8d4da7edf6df3f6d 2024-11-21T03:41:48,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/d6192e3973144862b330620cf00cd35e as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/d6192e3973144862b330620cf00cd35e 2024-11-21T03:41:48,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/d6192e3973144862b330620cf00cd35e, entries=200, sequenceid=199, filesize=38.8 K 2024-11-21T03:41:48,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/7ca7764c19d94f449e4f1a7c502945e4 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/7ca7764c19d94f449e4f1a7c502945e4 2024-11-21T03:41:48,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/7ca7764c19d94f449e4f1a7c502945e4, entries=150, sequenceid=199, filesize=11.9 K 2024-11-21T03:41:48,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/2638e58e7ed04a0c8d4da7edf6df3f6d as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/2638e58e7ed04a0c8d4da7edf6df3f6d 2024-11-21T03:41:48,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/2638e58e7ed04a0c8d4da7edf6df3f6d, entries=150, sequenceid=199, filesize=11.9 K 2024-11-21T03:41:48,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for f60eea9fbbad1295666bbf6cb6ce080c in 858ms, sequenceid=199, compaction requested=true 2024-11-21T03:41:48,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:48,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f60eea9fbbad1295666bbf6cb6ce080c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:41:48,320 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:48,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:48,320 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:48,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f60eea9fbbad1295666bbf6cb6ce080c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:41:48,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:48,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f60eea9fbbad1295666bbf6cb6ce080c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:41:48,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:48,321 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:48,321 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): f60eea9fbbad1295666bbf6cb6ce080c/B is initiating minor compaction (all files) 2024-11-21T03:41:48,321 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f60eea9fbbad1295666bbf6cb6ce080c/B in TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:48,321 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/4863f19a55ad4e5a9068ca1712207bd0, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/7e2fbdaa820f4a0f84218b71f24471e5, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/7ca7764c19d94f449e4f1a7c502945e4] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp, totalSize=35.9 K 2024-11-21T03:41:48,321 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:48,321 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): f60eea9fbbad1295666bbf6cb6ce080c/A is initiating minor compaction (all files) 2024-11-21T03:41:48,321 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f60eea9fbbad1295666bbf6cb6ce080c/A in TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:48,321 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 4863f19a55ad4e5a9068ca1712207bd0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732160502931 2024-11-21T03:41:48,321 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/de2297d7197e4974a0aa4aca3ec5c793, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/93b45344029c4b768de664e1399de782, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/d6192e3973144862b330620cf00cd35e] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp, totalSize=99.9 K 2024-11-21T03:41:48,321 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:48,322 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. files: [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/de2297d7197e4974a0aa4aca3ec5c793, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/93b45344029c4b768de664e1399de782, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/d6192e3973144862b330620cf00cd35e] 2024-11-21T03:41:48,322 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e2fbdaa820f4a0f84218b71f24471e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732160504088 2024-11-21T03:41:48,322 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting de2297d7197e4974a0aa4aca3ec5c793, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732160502931 2024-11-21T03:41:48,322 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93b45344029c4b768de664e1399de782, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732160504088 2024-11-21T03:41:48,322 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ca7764c19d94f449e4f1a7c502945e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732160506297 2024-11-21T03:41:48,322 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6192e3973144862b330620cf00cd35e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732160506297 2024-11-21T03:41:48,335 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:48,336 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f60eea9fbbad1295666bbf6cb6ce080c#B#compaction#348 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:48,337 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/1bb329e9f22e44ab9d59eaebac8ac532 is 50, key is test_row_0/B:col10/1732160506297/Put/seqid=0 2024-11-21T03:41:48,341 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411217487c3969ee64abd96227eaa0612d3e0_f60eea9fbbad1295666bbf6cb6ce080c store=[table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:48,343 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411217487c3969ee64abd96227eaa0612d3e0_f60eea9fbbad1295666bbf6cb6ce080c, store=[table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:48,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742235_1411 (size=12595) 2024-11-21T03:41:48,343 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217487c3969ee64abd96227eaa0612d3e0_f60eea9fbbad1295666bbf6cb6ce080c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:48,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742236_1412 (size=4469) 2024-11-21T03:41:48,348 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f60eea9fbbad1295666bbf6cb6ce080c#A#compaction#349 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:48,349 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/5cc7b510d2b14ff2a08721f3a2d1951b is 175, key is test_row_0/A:col10/1732160506297/Put/seqid=0 2024-11-21T03:41:48,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742237_1413 (size=31549) 2024-11-21T03:41:48,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T03:41:48,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:48,397 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing f60eea9fbbad1295666bbf6cb6ce080c 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:41:48,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219da44f1593864271ad01c8478b65893c_f60eea9fbbad1295666bbf6cb6ce080c is 50, key is test_row_0/A:col10/1732160507482/Put/seqid=0 2024-11-21T03:41:48,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742238_1414 (size=12304) 2024-11-21T03:41:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T03:41:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:48,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:48,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:48,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160568629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:48,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:48,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:48,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160568634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:48,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:48,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160568636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:48,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:48,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:48,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160568637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:48,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:48,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:48,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160568643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:48,748 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/1bb329e9f22e44ab9d59eaebac8ac532 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/1bb329e9f22e44ab9d59eaebac8ac532 2024-11-21T03:41:48,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:48,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160568747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,752 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:48,752 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f60eea9fbbad1295666bbf6cb6ce080c/B of f60eea9fbbad1295666bbf6cb6ce080c into 1bb329e9f22e44ab9d59eaebac8ac532(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:48,752 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:48,752 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., storeName=f60eea9fbbad1295666bbf6cb6ce080c/B, priority=13, startTime=1732160508320; duration=0sec 2024-11-21T03:41:48,752 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:48,752 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f60eea9fbbad1295666bbf6cb6ce080c:B 2024-11-21T03:41:48,752 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:48,753 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:48,753 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): f60eea9fbbad1295666bbf6cb6ce080c/C is initiating minor compaction (all files) 2024-11-21T03:41:48,753 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f60eea9fbbad1295666bbf6cb6ce080c/C in TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:48,753 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/ceafe454e7654a4b98faae8d5dfce2c3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/e2fc5efd5ac04d98a3a83095f7bbe65d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/2638e58e7ed04a0c8d4da7edf6df3f6d] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp, totalSize=35.9 K 2024-11-21T03:41:48,754 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting ceafe454e7654a4b98faae8d5dfce2c3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1732160502931 2024-11-21T03:41:48,754 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting e2fc5efd5ac04d98a3a83095f7bbe65d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732160504088 2024-11-21T03:41:48,755 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 2638e58e7ed04a0c8d4da7edf6df3f6d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732160506297 2024-11-21T03:41:48,758 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/5cc7b510d2b14ff2a08721f3a2d1951b as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/5cc7b510d2b14ff2a08721f3a2d1951b 2024-11-21T03:41:48,761 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f60eea9fbbad1295666bbf6cb6ce080c#C#compaction#351 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:48,762 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/b51bad733e4542b8aeb135bb4d95d760 is 50, key is test_row_0/C:col10/1732160506297/Put/seqid=0 2024-11-21T03:41:48,762 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f60eea9fbbad1295666bbf6cb6ce080c/A of f60eea9fbbad1295666bbf6cb6ce080c into 5cc7b510d2b14ff2a08721f3a2d1951b(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:48,762 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:48,762 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., storeName=f60eea9fbbad1295666bbf6cb6ce080c/A, priority=13, startTime=1732160508320; duration=0sec 2024-11-21T03:41:48,763 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:48,763 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f60eea9fbbad1295666bbf6cb6ce080c:A 2024-11-21T03:41:48,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:48,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160568757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:48,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742239_1415 (size=12595) 2024-11-21T03:41:48,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160568767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:48,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:48,811 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219da44f1593864271ad01c8478b65893c_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219da44f1593864271ad01c8478b65893c_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:48,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/fd9be9d47bd34b9da07fdd3a16674ee6, store: [table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:48,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/fd9be9d47bd34b9da07fdd3a16674ee6 is 175, key is test_row_0/A:col10/1732160507482/Put/seqid=0 2024-11-21T03:41:48,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742240_1416 (size=31105) 2024-11-21T03:41:48,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:48,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160568956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:48,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:48,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160568967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:48,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:48,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:48,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160568987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:48,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:48,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:49,172 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/b51bad733e4542b8aeb135bb4d95d760 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/b51bad733e4542b8aeb135bb4d95d760 2024-11-21T03:41:49,176 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f60eea9fbbad1295666bbf6cb6ce080c/C of f60eea9fbbad1295666bbf6cb6ce080c into b51bad733e4542b8aeb135bb4d95d760(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:49,176 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:49,176 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., storeName=f60eea9fbbad1295666bbf6cb6ce080c/C, priority=13, startTime=1732160508320; duration=0sec 2024-11-21T03:41:49,176 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:49,176 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f60eea9fbbad1295666bbf6cb6ce080c:C 2024-11-21T03:41:49,216 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/fd9be9d47bd34b9da07fdd3a16674ee6 2024-11-21T03:41:49,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/6052a8cc1e174f239935578f23b0bd42 is 50, key is test_row_0/B:col10/1732160507482/Put/seqid=0 2024-11-21T03:41:49,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742241_1417 (size=12151) 2024-11-21T03:41:49,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:49,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160569277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:49,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:49,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:49,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:49,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:49,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160569286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:49,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:49,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160569307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:49,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:49,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:49,309 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T03:41:49,624 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/6052a8cc1e174f239935578f23b0bd42 2024-11-21T03:41:49,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/0c8cd72c3bb54e5cb06b08c8b07215f7 is 50, key is test_row_0/C:col10/1732160507482/Put/seqid=0 2024-11-21T03:41:49,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742242_1418 (size=12151) 2024-11-21T03:41:49,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160569656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:49,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160569656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:49,662 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_1' in f60eea9fbbad1295666bbf6cb6ce080c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:41:49,662 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_2' in f60eea9fbbad1295666bbf6cb6ce080c of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:41:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:49,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:49,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160569797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:49,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:49,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:49,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:49,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:49,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160569807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:49,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:49,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:49,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:49,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160569817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:50,033 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/0c8cd72c3bb54e5cb06b08c8b07215f7 2024-11-21T03:41:50,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/fd9be9d47bd34b9da07fdd3a16674ee6 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/fd9be9d47bd34b9da07fdd3a16674ee6 2024-11-21T03:41:50,038 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/fd9be9d47bd34b9da07fdd3a16674ee6, entries=150, sequenceid=210, filesize=30.4 K 2024-11-21T03:41:50,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/6052a8cc1e174f239935578f23b0bd42 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/6052a8cc1e174f239935578f23b0bd42 2024-11-21T03:41:50,042 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/6052a8cc1e174f239935578f23b0bd42, entries=150, sequenceid=210, filesize=11.9 K 2024-11-21T03:41:50,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/0c8cd72c3bb54e5cb06b08c8b07215f7 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/0c8cd72c3bb54e5cb06b08c8b07215f7 2024-11-21T03:41:50,046 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/0c8cd72c3bb54e5cb06b08c8b07215f7, entries=150, sequenceid=210, filesize=11.9 K 2024-11-21T03:41:50,047 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f60eea9fbbad1295666bbf6cb6ce080c in 1650ms, sequenceid=210, compaction requested=false 2024-11-21T03:41:50,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:50,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:50,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-21T03:41:50,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-21T03:41:50,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-21T03:41:50,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7190 sec 2024-11-21T03:41:50,052 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.7230 sec 2024-11-21T03:41:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:50,820 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:41:50,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f60eea9fbbad1295666bbf6cb6ce080c 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T03:41:50,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121839188e3ffc2425d9fad5d5661594c50_f60eea9fbbad1295666bbf6cb6ce080c is 50, key is test_row_0/A:col10/1732160508627/Put/seqid=0 2024-11-21T03:41:50,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742243_1419 (size=14794) 2024-11-21T03:41:50,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160570855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:50,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:50,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:50,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:50,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160570867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:50,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:50,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:50,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:50,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160570868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:50,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:50,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:50,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:50,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:50,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160570977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:50,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:50,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:50,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:50,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160570977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:50,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:50,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:50,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:50,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:50,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:50,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160570987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:50,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:50,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:50,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:51,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160571187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:51,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:51,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160571187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:51,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:51,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:51,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:51,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:51,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:51,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160571197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:51,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:51,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:51,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:51,233 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:51,236 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121839188e3ffc2425d9fad5d5661594c50_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121839188e3ffc2425d9fad5d5661594c50_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:51,237 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/f2b697bbcca24b418f151df4da86d6f0, store: [table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:51,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/f2b697bbcca24b418f151df4da86d6f0 is 175, key is test_row_0/A:col10/1732160508627/Put/seqid=0 2024-11-21T03:41:51,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742244_1420 (size=39749) 2024-11-21T03:41:51,246 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/f2b697bbcca24b418f151df4da86d6f0 2024-11-21T03:41:51,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/e2ab722cadff493db09676205abcf6cd is 50, key is test_row_0/B:col10/1732160508627/Put/seqid=0 2024-11-21T03:41:51,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742245_1421 (size=12151) 2024-11-21T03:41:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T03:41:51,496 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:41:51,497 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:41:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-21T03:41:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T03:41:51,498 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:41:51,499 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:41:51,499 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:41:51,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160571496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:51,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160571496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:51,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:51,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:51,507 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:51,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160571507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T03:41:51,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T03:41:51,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:51,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:51,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:51,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:51,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:51,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/e2ab722cadff493db09676205abcf6cd 2024-11-21T03:41:51,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/f289d79037b7471ba9787046e609ae0d is 50, key is test_row_0/C:col10/1732160508627/Put/seqid=0 2024-11-21T03:41:51,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742246_1422 (size=12151) 2024-11-21T03:41:51,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37256 deadline: 1732160571667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:51,671 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_1' in f60eea9fbbad1295666bbf6cb6ce080c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:41:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:51,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37272 deadline: 1732160571686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:51,689 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fdc0a393186c,34631,1732160396889 for 'test_row_2' in f60eea9fbbad1295666bbf6cb6ce080c of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-21T03:41:51,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:51,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:51,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:51,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T03:41:51,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:51,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:51,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:51,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:51,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T03:41:51,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T03:41:51,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:51,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:51,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:51,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:51,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:51,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:41:52,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:52,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37270 deadline: 1732160572017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:52,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:52,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:52,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:52,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:52,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37218 deadline: 1732160572017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:52,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:52,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:52,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:52,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:41:52,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37318 deadline: 1732160572026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:52,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 , the old value is region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:52,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:41:52,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., hostname=fdc0a393186c,34631,1732160396889, seqNum=5 because the exception is null or not the one we care about 2024-11-21T03:41:52,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/f289d79037b7471ba9787046e609ae0d 2024-11-21T03:41:52,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/f2b697bbcca24b418f151df4da86d6f0 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/f2b697bbcca24b418f151df4da86d6f0 2024-11-21T03:41:52,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/f2b697bbcca24b418f151df4da86d6f0, entries=200, sequenceid=239, filesize=38.8 K 2024-11-21T03:41:52,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/e2ab722cadff493db09676205abcf6cd as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/e2ab722cadff493db09676205abcf6cd 2024-11-21T03:41:52,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/e2ab722cadff493db09676205abcf6cd, entries=150, sequenceid=239, filesize=11.9 K 2024-11-21T03:41:52,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/f289d79037b7471ba9787046e609ae0d as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/f289d79037b7471ba9787046e609ae0d 2024-11-21T03:41:52,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/f289d79037b7471ba9787046e609ae0d, entries=150, sequenceid=239, filesize=11.9 K 2024-11-21T03:41:52,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for f60eea9fbbad1295666bbf6cb6ce080c in 1260ms, sequenceid=239, compaction requested=true 2024-11-21T03:41:52,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:52,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f60eea9fbbad1295666bbf6cb6ce080c:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:41:52,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:52,080 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:52,080 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:52,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f60eea9fbbad1295666bbf6cb6ce080c:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:41:52,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:52,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f60eea9fbbad1295666bbf6cb6ce080c:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:41:52,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:52,081 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:52,081 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): f60eea9fbbad1295666bbf6cb6ce080c/A is initiating minor compaction (all files) 2024-11-21T03:41:52,081 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f60eea9fbbad1295666bbf6cb6ce080c/A in TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:52,081 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/5cc7b510d2b14ff2a08721f3a2d1951b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/fd9be9d47bd34b9da07fdd3a16674ee6, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/f2b697bbcca24b418f151df4da86d6f0] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp, totalSize=100.0 K 2024-11-21T03:41:52,081 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:52,081 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. files: [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/5cc7b510d2b14ff2a08721f3a2d1951b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/fd9be9d47bd34b9da07fdd3a16674ee6, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/f2b697bbcca24b418f151df4da86d6f0] 2024-11-21T03:41:52,081 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:52,081 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): f60eea9fbbad1295666bbf6cb6ce080c/B is initiating minor compaction (all files) 2024-11-21T03:41:52,081 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f60eea9fbbad1295666bbf6cb6ce080c/B in TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:52,082 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/1bb329e9f22e44ab9d59eaebac8ac532, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/6052a8cc1e174f239935578f23b0bd42, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/e2ab722cadff493db09676205abcf6cd] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp, totalSize=36.0 K 2024-11-21T03:41:52,082 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cc7b510d2b14ff2a08721f3a2d1951b, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732160506297 2024-11-21T03:41:52,082 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bb329e9f22e44ab9d59eaebac8ac532, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732160506297 2024-11-21T03:41:52,082 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd9be9d47bd34b9da07fdd3a16674ee6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732160507468 2024-11-21T03:41:52,082 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 6052a8cc1e174f239935578f23b0bd42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732160507468 2024-11-21T03:41:52,082 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2b697bbcca24b418f151df4da86d6f0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732160508627 2024-11-21T03:41:52,082 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting e2ab722cadff493db09676205abcf6cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732160508627 2024-11-21T03:41:52,094 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:52,095 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f60eea9fbbad1295666bbf6cb6ce080c#B#compaction#357 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:52,096 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/1d25d86ca4934e0885029bc464519073 is 50, key is test_row_0/B:col10/1732160508627/Put/seqid=0 2024-11-21T03:41:52,096 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121c18d7b371d854021af5a4e09da8ca1e1_f60eea9fbbad1295666bbf6cb6ce080c store=[table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:52,098 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121c18d7b371d854021af5a4e09da8ca1e1_f60eea9fbbad1295666bbf6cb6ce080c, store=[table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:52,098 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c18d7b371d854021af5a4e09da8ca1e1_f60eea9fbbad1295666bbf6cb6ce080c because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:52,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742247_1423 (size=12697) 2024-11-21T03:41:52,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742248_1424 (size=4469) 2024-11-21T03:41:52,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-21T03:41:52,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:52,109 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing f60eea9fbbad1295666bbf6cb6ce080c 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-21T03:41:52,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214062afdcaa6045b7bb649a8aa788836f_f60eea9fbbad1295666bbf6cb6ce080c is 50, key is test_row_0/A:col10/1732160510854/Put/seqid=0 2024-11-21T03:41:52,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742249_1425 (size=12304) 2024-11-21T03:41:52,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T03:41:52,504 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/1d25d86ca4934e0885029bc464519073 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/1d25d86ca4934e0885029bc464519073 2024-11-21T03:41:52,505 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f60eea9fbbad1295666bbf6cb6ce080c#A#compaction#358 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:52,506 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/b374eca1045442fca74434b0900d80b3 is 175, key is test_row_0/A:col10/1732160508627/Put/seqid=0 2024-11-21T03:41:52,508 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f60eea9fbbad1295666bbf6cb6ce080c/B of f60eea9fbbad1295666bbf6cb6ce080c into 1d25d86ca4934e0885029bc464519073(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:52,508 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:52,508 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., storeName=f60eea9fbbad1295666bbf6cb6ce080c/B, priority=13, startTime=1732160512080; duration=0sec 2024-11-21T03:41:52,508 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:41:52,508 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f60eea9fbbad1295666bbf6cb6ce080c:B 2024-11-21T03:41:52,509 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T03:41:52,509 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T03:41:52,509 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): f60eea9fbbad1295666bbf6cb6ce080c/C is initiating minor compaction (all files) 2024-11-21T03:41:52,510 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f60eea9fbbad1295666bbf6cb6ce080c/C in TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:52,510 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/b51bad733e4542b8aeb135bb4d95d760, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/0c8cd72c3bb54e5cb06b08c8b07215f7, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/f289d79037b7471ba9787046e609ae0d] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp, totalSize=36.0 K 2024-11-21T03:41:52,510 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting b51bad733e4542b8aeb135bb4d95d760, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1732160506297 2024-11-21T03:41:52,510 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c8cd72c3bb54e5cb06b08c8b07215f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1732160507468 2024-11-21T03:41:52,510 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting f289d79037b7471ba9787046e609ae0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732160508627 2024-11-21T03:41:52,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:52,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742250_1426 (size=31651) 2024-11-21T03:41:52,525 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214062afdcaa6045b7bb649a8aa788836f_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214062afdcaa6045b7bb649a8aa788836f_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:52,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/e404609eef40481ea3d68a314eec308d, store: [table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:52,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/e404609eef40481ea3d68a314eec308d is 175, key is test_row_0/A:col10/1732160510854/Put/seqid=0 2024-11-21T03:41:52,529 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/b374eca1045442fca74434b0900d80b3 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/b374eca1045442fca74434b0900d80b3 2024-11-21T03:41:52,530 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f60eea9fbbad1295666bbf6cb6ce080c#C#compaction#360 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:41:52,530 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/7dda95df42be4fa7870b25d4f3b93a15 is 50, key is test_row_0/C:col10/1732160508627/Put/seqid=0 2024-11-21T03:41:52,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742251_1427 (size=31105) 2024-11-21T03:41:52,532 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/e404609eef40481ea3d68a314eec308d 2024-11-21T03:41:52,538 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f60eea9fbbad1295666bbf6cb6ce080c/A of f60eea9fbbad1295666bbf6cb6ce080c into b374eca1045442fca74434b0900d80b3(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:52,538 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:52,538 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., storeName=f60eea9fbbad1295666bbf6cb6ce080c/A, priority=13, startTime=1732160512080; duration=0sec 2024-11-21T03:41:52,538 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:52,538 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f60eea9fbbad1295666bbf6cb6ce080c:A 2024-11-21T03:41:52,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/db2449220fd24320978f1c0e6f0cc2df is 50, key is test_row_0/B:col10/1732160510854/Put/seqid=0 2024-11-21T03:41:52,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742252_1428 (size=12697) 2024-11-21T03:41:52,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742253_1429 (size=12151) 2024-11-21T03:41:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T03:41:52,687 INFO [Thread-1633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1633. 2024-11-21T03:41:52,687 DEBUG [Thread-1633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:41:52,687 DEBUG [Thread-1633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:52,687 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:41:52,687 DEBUG [Thread-1633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:52,688 INFO [Thread-1635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1635. 2024-11-21T03:41:52,688 DEBUG [Thread-1635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:41:52,688 DEBUG [Thread-1635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:52,688 DEBUG [Thread-1635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:52,688 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:41:52,690 INFO [Thread-1637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1637. 2024-11-21T03:41:52,690 DEBUG [Thread-1637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:41:52,690 DEBUG [Thread-1637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:52,690 DEBUG [Thread-1637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:52,690 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:41:52,692 INFO [Thread-1629 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1629. 2024-11-21T03:41:52,692 DEBUG [Thread-1629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:41:52,692 DEBUG [Thread-1629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:52,692 DEBUG [Thread-1629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:52,692 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:41:52,693 INFO [Thread-1631 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1631. 2024-11-21T03:41:52,694 DEBUG [Thread-1631 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:41:52,694 DEBUG [Thread-1631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:52,694 DEBUG [Thread-1631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:52,694 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:41:52,981 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/db2449220fd24320978f1c0e6f0cc2df 2024-11-21T03:41:52,986 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/7dda95df42be4fa7870b25d4f3b93a15 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/7dda95df42be4fa7870b25d4f3b93a15 2024-11-21T03:41:52,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/4958dd0f22264d5abe0a907ae21e96fd is 50, key is test_row_0/C:col10/1732160510854/Put/seqid=0 2024-11-21T03:41:52,993 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f60eea9fbbad1295666bbf6cb6ce080c/C of f60eea9fbbad1295666bbf6cb6ce080c into 7dda95df42be4fa7870b25d4f3b93a15(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:41:52,993 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:52,993 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., storeName=f60eea9fbbad1295666bbf6cb6ce080c/C, priority=13, startTime=1732160512080; duration=0sec 2024-11-21T03:41:52,993 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:41:52,993 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f60eea9fbbad1295666bbf6cb6ce080c:C 2024-11-21T03:41:52,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742254_1430 (size=12151) 2024-11-21T03:41:53,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:53,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. as already flushing 2024-11-21T03:41:53,028 INFO [Thread-1622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1622. 2024-11-21T03:41:53,028 DEBUG [Thread-1622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:41:53,028 DEBUG [Thread-1622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:53,028 DEBUG [Thread-1622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:53,028 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:41:53,038 INFO [Thread-1624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1624. 2024-11-21T03:41:53,038 DEBUG [Thread-1624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:41:53,038 DEBUG [Thread-1624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:53,038 DEBUG [Thread-1624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:53,038 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:41:53,049 INFO [Thread-1620 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1620. 2024-11-21T03:41:53,049 DEBUG [Thread-1620 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:41:53,049 DEBUG [Thread-1620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:53,049 DEBUG [Thread-1620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:53,049 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:41:53,397 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/4958dd0f22264d5abe0a907ae21e96fd 2024-11-21T03:41:53,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/e404609eef40481ea3d68a314eec308d as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/e404609eef40481ea3d68a314eec308d 2024-11-21T03:41:53,409 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/e404609eef40481ea3d68a314eec308d, entries=150, sequenceid=249, filesize=30.4 K 2024-11-21T03:41:53,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/db2449220fd24320978f1c0e6f0cc2df as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/db2449220fd24320978f1c0e6f0cc2df 2024-11-21T03:41:53,415 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/db2449220fd24320978f1c0e6f0cc2df, entries=150, sequenceid=249, filesize=11.9 K 2024-11-21T03:41:53,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/4958dd0f22264d5abe0a907ae21e96fd as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/4958dd0f22264d5abe0a907ae21e96fd 2024-11-21T03:41:53,419 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/4958dd0f22264d5abe0a907ae21e96fd, entries=150, sequenceid=249, filesize=11.9 K 2024-11-21T03:41:53,420 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=20.13 KB/20610 for f60eea9fbbad1295666bbf6cb6ce080c in 1310ms, sequenceid=249, compaction requested=false 2024-11-21T03:41:53,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for f60eea9fbbad1295666bbf6cb6ce080c: 2024-11-21T03:41:53,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:53,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-21T03:41:53,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-21T03:41:53,423 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-21T03:41:53,423 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9220 sec 2024-11-21T03:41:53,424 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.9260 sec 2024-11-21T03:41:53,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T03:41:53,658 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:41:55,192 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T03:41:55,712 INFO [Thread-1618 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1618. 2024-11-21T03:41:55,712 INFO [Thread-1626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1626. 2024-11-21T03:41:55,712 DEBUG [Thread-1618 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:41:55,712 DEBUG [Thread-1626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T03:41:55,712 DEBUG [Thread-1618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:55,712 DEBUG [Thread-1626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:55,713 DEBUG [Thread-1618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:55,713 DEBUG [Thread-1626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:55,713 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:41:55,713 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:41:55,714 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T03:41:55,714 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-11-21T03:41:55,714 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-21T03:41:55,714 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-21T03:41:55,714 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-21T03:41:55,714 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-21T03:41:55,714 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T03:41:55,714 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T03:41:55,714 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1522 2024-11-21T03:41:55,714 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4565 rows 2024-11-21T03:41:55,714 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1523 2024-11-21T03:41:55,714 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4566 rows 2024-11-21T03:41:55,714 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1520 2024-11-21T03:41:55,714 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4558 rows 2024-11-21T03:41:55,714 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1531 2024-11-21T03:41:55,714 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4587 rows 2024-11-21T03:41:55,714 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1527 2024-11-21T03:41:55,714 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4578 rows 2024-11-21T03:41:55,715 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T03:41:55,715 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T03:41:55,715 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:55,715 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:41:55,715 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T03:41:55,716 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T03:41:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T03:41:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T03:41:55,719 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732160515718"}]},"ts":"1732160515718"} 2024-11-21T03:41:55,720 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T03:41:55,720 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T03:41:55,721 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T03:41:55,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f60eea9fbbad1295666bbf6cb6ce080c, UNASSIGN}] 2024-11-21T03:41:55,723 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f60eea9fbbad1295666bbf6cb6ce080c, UNASSIGN 2024-11-21T03:41:55,724 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=f60eea9fbbad1295666bbf6cb6ce080c, regionState=CLOSING, regionLocation=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:55,725 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f60eea9fbbad1295666bbf6cb6ce080c, UNASSIGN because future has completed 2024-11-21T03:41:55,725 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T03:41:55,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; CloseRegionProcedure f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889}] 2024-11-21T03:41:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T03:41:55,879 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(122): Close f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:55,879 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T03:41:55,879 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1722): Closing f60eea9fbbad1295666bbf6cb6ce080c, disabling compactions & flushes 2024-11-21T03:41:55,879 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:55,879 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:55,879 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. after waiting 0 ms 2024-11-21T03:41:55,879 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:55,879 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(2902): Flushing f60eea9fbbad1295666bbf6cb6ce080c 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-21T03:41:55,890 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112155455130e6e2411a9cd2d00172a3adc8_f60eea9fbbad1295666bbf6cb6ce080c is 50, key is test_row_0/A:col10/1732160513047/Put/seqid=0 2024-11-21T03:41:55,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742255_1431 (size=12454) 2024-11-21T03:41:56,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T03:41:56,296 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:56,305 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112155455130e6e2411a9cd2d00172a3adc8_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112155455130e6e2411a9cd2d00172a3adc8_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:56,306 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/b8a8545070c54322825849dfaf07212d, store: [table=TestAcidGuarantees family=A region=f60eea9fbbad1295666bbf6cb6ce080c] 2024-11-21T03:41:56,307 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/b8a8545070c54322825849dfaf07212d is 175, key is test_row_0/A:col10/1732160513047/Put/seqid=0 2024-11-21T03:41:56,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742256_1432 (size=31255) 2024-11-21T03:41:56,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T03:41:56,711 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=260, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/b8a8545070c54322825849dfaf07212d 2024-11-21T03:41:56,718 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/734ce53e886444059d1886732d29f115 is 50, key is test_row_0/B:col10/1732160513047/Put/seqid=0 2024-11-21T03:41:56,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742257_1433 (size=12301) 2024-11-21T03:41:56,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T03:41:57,124 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/734ce53e886444059d1886732d29f115 2024-11-21T03:41:57,137 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/ec4c21b1872a415485c1e0308d7a362a is 50, key is test_row_0/C:col10/1732160513047/Put/seqid=0 2024-11-21T03:41:57,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742258_1434 (size=12301) 2024-11-21T03:41:57,541 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/ec4c21b1872a415485c1e0308d7a362a 2024-11-21T03:41:57,548 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/A/b8a8545070c54322825849dfaf07212d as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/b8a8545070c54322825849dfaf07212d 2024-11-21T03:41:57,553 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/b8a8545070c54322825849dfaf07212d, entries=150, sequenceid=260, filesize=30.5 K 2024-11-21T03:41:57,554 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/B/734ce53e886444059d1886732d29f115 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/734ce53e886444059d1886732d29f115 2024-11-21T03:41:57,560 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/734ce53e886444059d1886732d29f115, entries=150, sequenceid=260, filesize=12.0 K 2024-11-21T03:41:57,560 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/.tmp/C/ec4c21b1872a415485c1e0308d7a362a as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/ec4c21b1872a415485c1e0308d7a362a 2024-11-21T03:41:57,565 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/ec4c21b1872a415485c1e0308d7a362a, entries=150, sequenceid=260, filesize=12.0 K 2024-11-21T03:41:57,566 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for f60eea9fbbad1295666bbf6cb6ce080c in 1687ms, sequenceid=260, compaction requested=true 2024-11-21T03:41:57,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/642b47f67c1548cc842addaa323cd83c, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/a803a1eca6dc4a1f91433e819bfe3b22, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/2b3f6ae5239e4a07a9246a9dd508b212, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/8db21980d5544c7db9ddd2cc607b2648, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/2833d89ed1a74c43a19e7f3809018273, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/013e90b5d81e454e816a3292014fda3c, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/a919f5770f98460d9f83adc6408c7c6d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/e1085fbe226a49619a12d592384da3d9, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/995adb9eb189485eae96e654801c3a81, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/0a400ce1976d455482b9d1c5ee5711e5, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/de2297d7197e4974a0aa4aca3ec5c793, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/93b45344029c4b768de664e1399de782, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/d6192e3973144862b330620cf00cd35e, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/5cc7b510d2b14ff2a08721f3a2d1951b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/fd9be9d47bd34b9da07fdd3a16674ee6, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/f2b697bbcca24b418f151df4da86d6f0] to archive 2024-11-21T03:41:57,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:41:57,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/642b47f67c1548cc842addaa323cd83c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/642b47f67c1548cc842addaa323cd83c 2024-11-21T03:41:57,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/a803a1eca6dc4a1f91433e819bfe3b22 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/a803a1eca6dc4a1f91433e819bfe3b22 2024-11-21T03:41:57,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/2b3f6ae5239e4a07a9246a9dd508b212 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/2b3f6ae5239e4a07a9246a9dd508b212 2024-11-21T03:41:57,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/8db21980d5544c7db9ddd2cc607b2648 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/8db21980d5544c7db9ddd2cc607b2648 2024-11-21T03:41:57,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/2833d89ed1a74c43a19e7f3809018273 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/2833d89ed1a74c43a19e7f3809018273 2024-11-21T03:41:57,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/013e90b5d81e454e816a3292014fda3c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/013e90b5d81e454e816a3292014fda3c 2024-11-21T03:41:57,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/a919f5770f98460d9f83adc6408c7c6d to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/a919f5770f98460d9f83adc6408c7c6d 2024-11-21T03:41:57,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/e1085fbe226a49619a12d592384da3d9 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/e1085fbe226a49619a12d592384da3d9 2024-11-21T03:41:57,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/995adb9eb189485eae96e654801c3a81 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/995adb9eb189485eae96e654801c3a81 2024-11-21T03:41:57,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/0a400ce1976d455482b9d1c5ee5711e5 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/0a400ce1976d455482b9d1c5ee5711e5 2024-11-21T03:41:57,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/de2297d7197e4974a0aa4aca3ec5c793 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/de2297d7197e4974a0aa4aca3ec5c793 2024-11-21T03:41:57,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/93b45344029c4b768de664e1399de782 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/93b45344029c4b768de664e1399de782 2024-11-21T03:41:57,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/d6192e3973144862b330620cf00cd35e to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/d6192e3973144862b330620cf00cd35e 2024-11-21T03:41:57,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/5cc7b510d2b14ff2a08721f3a2d1951b to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/5cc7b510d2b14ff2a08721f3a2d1951b 2024-11-21T03:41:57,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/fd9be9d47bd34b9da07fdd3a16674ee6 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/fd9be9d47bd34b9da07fdd3a16674ee6 2024-11-21T03:41:57,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/f2b697bbcca24b418f151df4da86d6f0 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/f2b697bbcca24b418f151df4da86d6f0 2024-11-21T03:41:57,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/c6a524c26efb456b91ede14a446ab1da, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/34ef384b66204fbe9e53d29d097e48d4, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/b6839741f1ad499196ae6da8938989d5, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/01f5634fdaa04f8fbcb875d631a00ebc, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/c2dfa996092341fe8edc56b0485af8f3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/8e43e5069773422284159a183b01e545, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/550f473b5836488cbc3934739b6e2381, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/81d6621bf590477793faa83040f31194, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/48899da4bcaa4cc5849cffeb0459845b, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/4863f19a55ad4e5a9068ca1712207bd0, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/149c1dbfbf5b49c382116a3aa9869982, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/7e2fbdaa820f4a0f84218b71f24471e5, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/1bb329e9f22e44ab9d59eaebac8ac532, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/7ca7764c19d94f449e4f1a7c502945e4, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/6052a8cc1e174f239935578f23b0bd42, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/e2ab722cadff493db09676205abcf6cd] to archive 2024-11-21T03:41:57,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:41:57,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/c6a524c26efb456b91ede14a446ab1da to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/c6a524c26efb456b91ede14a446ab1da 2024-11-21T03:41:57,606 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/34ef384b66204fbe9e53d29d097e48d4 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/34ef384b66204fbe9e53d29d097e48d4 2024-11-21T03:41:57,608 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/b6839741f1ad499196ae6da8938989d5 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/b6839741f1ad499196ae6da8938989d5 2024-11-21T03:41:57,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/01f5634fdaa04f8fbcb875d631a00ebc to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/01f5634fdaa04f8fbcb875d631a00ebc 2024-11-21T03:41:57,611 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/c2dfa996092341fe8edc56b0485af8f3 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/c2dfa996092341fe8edc56b0485af8f3 2024-11-21T03:41:57,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/8e43e5069773422284159a183b01e545 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/8e43e5069773422284159a183b01e545 2024-11-21T03:41:57,615 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/550f473b5836488cbc3934739b6e2381 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/550f473b5836488cbc3934739b6e2381 2024-11-21T03:41:57,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/81d6621bf590477793faa83040f31194 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/81d6621bf590477793faa83040f31194 2024-11-21T03:41:57,618 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/48899da4bcaa4cc5849cffeb0459845b to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/48899da4bcaa4cc5849cffeb0459845b 2024-11-21T03:41:57,620 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/4863f19a55ad4e5a9068ca1712207bd0 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/4863f19a55ad4e5a9068ca1712207bd0 2024-11-21T03:41:57,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/149c1dbfbf5b49c382116a3aa9869982 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/149c1dbfbf5b49c382116a3aa9869982 2024-11-21T03:41:57,623 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/7e2fbdaa820f4a0f84218b71f24471e5 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/7e2fbdaa820f4a0f84218b71f24471e5 2024-11-21T03:41:57,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/1bb329e9f22e44ab9d59eaebac8ac532 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/1bb329e9f22e44ab9d59eaebac8ac532 2024-11-21T03:41:57,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/7ca7764c19d94f449e4f1a7c502945e4 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/7ca7764c19d94f449e4f1a7c502945e4 2024-11-21T03:41:57,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/6052a8cc1e174f239935578f23b0bd42 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/6052a8cc1e174f239935578f23b0bd42 2024-11-21T03:41:57,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/e2ab722cadff493db09676205abcf6cd to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/e2ab722cadff493db09676205abcf6cd 2024-11-21T03:41:57,632 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/c535eadae0ee47e2be546c5c9f86a669, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/f05e2244b65e4240b00f3a86b1046835, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/95896baa845b42e293f17ed84bbf6003, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/ca4c0d85548845789182fc7bc86260bd, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/91ed60e834454e8ea01edd0b7abcfd4d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/133fff001a964351aeb246115fa3d4a7, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/cb82fc47a77f44bbab1f9c14ceda70e5, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/8930bdbb178540bd8fb10d5ea49d70fe, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/d60f7aa07ca44880b00f619a7af7f82e, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/ceafe454e7654a4b98faae8d5dfce2c3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/ef63fa31fbd54a7d8be8097c7b4b4143, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/e2fc5efd5ac04d98a3a83095f7bbe65d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/b51bad733e4542b8aeb135bb4d95d760, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/2638e58e7ed04a0c8d4da7edf6df3f6d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/0c8cd72c3bb54e5cb06b08c8b07215f7, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/f289d79037b7471ba9787046e609ae0d] to archive 2024-11-21T03:41:57,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T03:41:57,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/c535eadae0ee47e2be546c5c9f86a669 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/c535eadae0ee47e2be546c5c9f86a669 2024-11-21T03:41:57,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/f05e2244b65e4240b00f3a86b1046835 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/f05e2244b65e4240b00f3a86b1046835 2024-11-21T03:41:57,638 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/95896baa845b42e293f17ed84bbf6003 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/95896baa845b42e293f17ed84bbf6003 2024-11-21T03:41:57,638 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/ca4c0d85548845789182fc7bc86260bd to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/ca4c0d85548845789182fc7bc86260bd 2024-11-21T03:41:57,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/91ed60e834454e8ea01edd0b7abcfd4d to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/91ed60e834454e8ea01edd0b7abcfd4d 2024-11-21T03:41:57,640 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/133fff001a964351aeb246115fa3d4a7 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/133fff001a964351aeb246115fa3d4a7 2024-11-21T03:41:57,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/cb82fc47a77f44bbab1f9c14ceda70e5 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/cb82fc47a77f44bbab1f9c14ceda70e5 2024-11-21T03:41:57,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/8930bdbb178540bd8fb10d5ea49d70fe to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/8930bdbb178540bd8fb10d5ea49d70fe 2024-11-21T03:41:57,642 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/d60f7aa07ca44880b00f619a7af7f82e to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/d60f7aa07ca44880b00f619a7af7f82e 2024-11-21T03:41:57,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/ceafe454e7654a4b98faae8d5dfce2c3 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/ceafe454e7654a4b98faae8d5dfce2c3 2024-11-21T03:41:57,643 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/ef63fa31fbd54a7d8be8097c7b4b4143 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/ef63fa31fbd54a7d8be8097c7b4b4143 2024-11-21T03:41:57,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/e2fc5efd5ac04d98a3a83095f7bbe65d to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/e2fc5efd5ac04d98a3a83095f7bbe65d 2024-11-21T03:41:57,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/b51bad733e4542b8aeb135bb4d95d760 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/b51bad733e4542b8aeb135bb4d95d760 2024-11-21T03:41:57,645 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/2638e58e7ed04a0c8d4da7edf6df3f6d to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/2638e58e7ed04a0c8d4da7edf6df3f6d 2024-11-21T03:41:57,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/0c8cd72c3bb54e5cb06b08c8b07215f7 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/0c8cd72c3bb54e5cb06b08c8b07215f7 2024-11-21T03:41:57,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/f289d79037b7471ba9787046e609ae0d to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/f289d79037b7471ba9787046e609ae0d 2024-11-21T03:41:57,650 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/recovered.edits/263.seqid, newMaxSeqId=263, maxSeqId=4 2024-11-21T03:41:57,651 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. 2024-11-21T03:41:57,651 DEBUG [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1676): Region close journal for f60eea9fbbad1295666bbf6cb6ce080c: Waiting for close lock at 1732160515879Running coprocessor pre-close hooks at 1732160515879Disabling compacts and flushes for region at 1732160515879Disabling writes for close at 1732160515879Obtaining lock to block concurrent updates at 1732160515879Preparing flush snapshotting stores in f60eea9fbbad1295666bbf6cb6ce080c at 1732160515879Finished memstore snapshotting TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732160515880 (+1 ms)Flushing stores of TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c. at 1732160515882 (+2 ms)Flushing f60eea9fbbad1295666bbf6cb6ce080c/A: creating writer at 1732160515882Flushing mob file f60eea9fbbad1295666bbf6cb6ce080c/A: appending metadata at 1732160515890 (+8 ms)Flushing mob file f60eea9fbbad1295666bbf6cb6ce080c/A: closing flushed file at 1732160515890Flushing f60eea9fbbad1295666bbf6cb6ce080c/A: appending metadata at 1732160516306 (+416 ms)Flushing f60eea9fbbad1295666bbf6cb6ce080c/A: closing flushed file at 1732160516306Flushing f60eea9fbbad1295666bbf6cb6ce080c/B: creating writer at 1732160516715 (+409 ms)Flushing f60eea9fbbad1295666bbf6cb6ce080c/B: appending metadata at 1732160516717 (+2 ms)Flushing f60eea9fbbad1295666bbf6cb6ce080c/B: closing flushed file at 1732160516717Flushing f60eea9fbbad1295666bbf6cb6ce080c/C: creating writer at 1732160517131 (+414 ms)Flushing f60eea9fbbad1295666bbf6cb6ce080c/C: appending metadata at 1732160517136 (+5 ms)Flushing f60eea9fbbad1295666bbf6cb6ce080c/C: closing flushed file at 1732160517136Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30eccf08: reopening flushed file at 1732160517547 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a973710: reopening flushed file at 1732160517553 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d71a493: reopening flushed file at 1732160517560 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for f60eea9fbbad1295666bbf6cb6ce080c in 1687ms, sequenceid=260, compaction requested=true at 1732160517566 (+6 ms)Writing region close event to WAL at 1732160517648 (+82 ms)Running coprocessor post-close hooks at 1732160517651 (+3 ms)Closed at 1732160517651 2024-11-21T03:41:57,653 INFO [RS_CLOSE_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(157): Closed f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:57,653 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=f60eea9fbbad1295666bbf6cb6ce080c, regionState=CLOSED 2024-11-21T03:41:57,654 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; CloseRegionProcedure f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 because future has completed 2024-11-21T03:41:57,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-21T03:41:57,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseRegionProcedure f60eea9fbbad1295666bbf6cb6ce080c, server=fdc0a393186c,34631,1732160396889 in 1.9300 sec 2024-11-21T03:41:57,658 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-21T03:41:57,658 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f60eea9fbbad1295666bbf6cb6ce080c, UNASSIGN in 1.9340 sec 2024-11-21T03:41:57,659 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-21T03:41:57,659 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9370 sec 2024-11-21T03:41:57,660 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732160517660"}]},"ts":"1732160517660"} 2024-11-21T03:41:57,661 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T03:41:57,661 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T03:41:57,662 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9450 sec 2024-11-21T03:41:57,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-21T03:41:57,868 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:41:57,870 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T03:41:57,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:41:57,875 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=121, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:41:57,877 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=121, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:41:57,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T03:41:57,880 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:57,884 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A, FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B, FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C, FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/recovered.edits] 2024-11-21T03:41:57,889 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/b374eca1045442fca74434b0900d80b3 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/b374eca1045442fca74434b0900d80b3 2024-11-21T03:41:57,890 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/b8a8545070c54322825849dfaf07212d to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/b8a8545070c54322825849dfaf07212d 2024-11-21T03:41:57,892 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/e404609eef40481ea3d68a314eec308d to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/A/e404609eef40481ea3d68a314eec308d 2024-11-21T03:41:57,896 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/1d25d86ca4934e0885029bc464519073 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/1d25d86ca4934e0885029bc464519073 2024-11-21T03:41:57,897 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/734ce53e886444059d1886732d29f115 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/734ce53e886444059d1886732d29f115 2024-11-21T03:41:57,898 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/db2449220fd24320978f1c0e6f0cc2df to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/B/db2449220fd24320978f1c0e6f0cc2df 2024-11-21T03:41:57,900 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/4958dd0f22264d5abe0a907ae21e96fd to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/4958dd0f22264d5abe0a907ae21e96fd 2024-11-21T03:41:57,901 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/7dda95df42be4fa7870b25d4f3b93a15 to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/7dda95df42be4fa7870b25d4f3b93a15 2024-11-21T03:41:57,902 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/ec4c21b1872a415485c1e0308d7a362a to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/C/ec4c21b1872a415485c1e0308d7a362a 2024-11-21T03:41:57,904 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/recovered.edits/263.seqid to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c/recovered.edits/263.seqid 2024-11-21T03:41:57,905 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:57,905 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T03:41:57,905 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T03:41:57,906 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-21T03:41:57,908 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210c38e1ef632649ba9865781066df12fc_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210c38e1ef632649ba9865781066df12fc_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:57,909 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214062afdcaa6045b7bb649a8aa788836f_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214062afdcaa6045b7bb649a8aa788836f_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:57,910 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121535ff22b892f49d6a5683c3498742b59_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121535ff22b892f49d6a5683c3498742b59_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:57,911 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112155455130e6e2411a9cd2d00172a3adc8_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112155455130e6e2411a9cd2d00172a3adc8_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:57,912 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216e787c43b65c4cc69c464d2989e48ce6_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216e787c43b65c4cc69c464d2989e48ce6_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:57,913 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217bf98e31635d479eb624ecb79e8fc35d_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217bf98e31635d479eb624ecb79e8fc35d_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:57,914 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121839188e3ffc2425d9fad5d5661594c50_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121839188e3ffc2425d9fad5d5661594c50_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:57,915 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218b6a7f5b1c9c49df93bc6c0891eb859b_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218b6a7f5b1c9c49df93bc6c0891eb859b_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:57,916 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218fb9d454c1904a37a6745c6fe3d1f9b1_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218fb9d454c1904a37a6745c6fe3d1f9b1_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:57,917 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219da44f1593864271ad01c8478b65893c_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219da44f1593864271ad01c8478b65893c_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:57,918 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a4339b3a6250440fbd58c8dd301a42b7_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a4339b3a6250440fbd58c8dd301a42b7_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:57,919 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b35badc7e7de40a491cea95b72249d07_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b35badc7e7de40a491cea95b72249d07_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:57,920 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bd0fa9b3fd4b436a97dc5b2251cacbd9_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121bd0fa9b3fd4b436a97dc5b2251cacbd9_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:57,921 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c56a341ec30e4da6a902b0472940e3a8_f60eea9fbbad1295666bbf6cb6ce080c to hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c56a341ec30e4da6a902b0472940e3a8_f60eea9fbbad1295666bbf6cb6ce080c 2024-11-21T03:41:57,921 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T03:41:57,923 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=121, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:41:57,924 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T03:41:57,926 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T03:41:57,927 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=121, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:41:57,927 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T03:41:57,927 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732160517927"}]},"ts":"9223372036854775807"} 2024-11-21T03:41:57,928 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T03:41:57,928 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f60eea9fbbad1295666bbf6cb6ce080c, NAME => 'TestAcidGuarantees,,1732160489524.f60eea9fbbad1295666bbf6cb6ce080c.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T03:41:57,928 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T03:41:57,928 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732160517928"}]},"ts":"9223372036854775807"} 2024-11-21T03:41:57,930 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T03:41:57,930 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=121, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T03:41:57,931 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 60 msec 2024-11-21T03:41:57,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-21T03:41:57,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T03:41:57,990 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:41:57,999 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=239 (was 238) - Thread LEAK? -, OpenFileDescriptor=457 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=449 (was 491), ProcessCount=11 (was 11), AvailableMemoryMB=6559 (was 6594) 2024-11-21T03:41:58,008 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=239, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=449, ProcessCount=11, AvailableMemoryMB=6559 2024-11-21T03:41:58,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T03:41:58,009 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T03:41:58,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=122, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T03:41:58,026 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T03:41:58,026 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:58,026 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 122 2024-11-21T03:41:58,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-21T03:41:58,027 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T03:41:58,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742259_1435 (size=959) 2024-11-21T03:41:58,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-21T03:41:58,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-21T03:41:58,442 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0 2024-11-21T03:41:58,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742260_1436 (size=53) 2024-11-21T03:41:58,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-21T03:41:58,852 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:41:58,853 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e6d0058d8dbcf6c4f9d7eedab2fc5a3e, disabling compactions & flushes 2024-11-21T03:41:58,853 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:41:58,853 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:41:58,853 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. after waiting 0 ms 2024-11-21T03:41:58,853 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:41:58,853 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:41:58,853 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e6d0058d8dbcf6c4f9d7eedab2fc5a3e: Waiting for close lock at 1732160518853Disabling compacts and flushes for region at 1732160518853Disabling writes for close at 1732160518853Writing region close event to WAL at 1732160518853Closed at 1732160518853 2024-11-21T03:41:58,856 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T03:41:58,857 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732160518856"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732160518856"}]},"ts":"1732160518856"} 2024-11-21T03:41:58,860 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T03:41:58,861 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T03:41:58,861 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732160518861"}]},"ts":"1732160518861"} 2024-11-21T03:41:58,863 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T03:41:58,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, ASSIGN}] 2024-11-21T03:41:58,864 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, ASSIGN 2024-11-21T03:41:58,865 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, ASSIGN; state=OFFLINE, location=fdc0a393186c,34631,1732160396889; forceNewPlan=false, retain=false 2024-11-21T03:41:59,016 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, regionState=OPENING, regionLocation=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:59,019 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, ASSIGN because future has completed 2024-11-21T03:41:59,020 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; OpenRegionProcedure e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889}] 2024-11-21T03:41:59,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-21T03:41:59,178 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:41:59,178 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7752): Opening region: {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} 2024-11-21T03:41:59,180 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e6d0058d8dbcf6c4f9d7eedab2fc5a3e 2024-11-21T03:41:59,180 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T03:41:59,180 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7794): checking encryption for e6d0058d8dbcf6c4f9d7eedab2fc5a3e 2024-11-21T03:41:59,180 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7797): checking classloading for e6d0058d8dbcf6c4f9d7eedab2fc5a3e 2024-11-21T03:41:59,183 INFO [StoreOpener-e6d0058d8dbcf6c4f9d7eedab2fc5a3e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e6d0058d8dbcf6c4f9d7eedab2fc5a3e 2024-11-21T03:41:59,186 INFO [StoreOpener-e6d0058d8dbcf6c4f9d7eedab2fc5a3e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6d0058d8dbcf6c4f9d7eedab2fc5a3e columnFamilyName A 2024-11-21T03:41:59,186 DEBUG [StoreOpener-e6d0058d8dbcf6c4f9d7eedab2fc5a3e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:59,187 INFO [StoreOpener-e6d0058d8dbcf6c4f9d7eedab2fc5a3e-1 {}] regionserver.HStore(327): Store=e6d0058d8dbcf6c4f9d7eedab2fc5a3e/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:41:59,187 INFO [StoreOpener-e6d0058d8dbcf6c4f9d7eedab2fc5a3e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e6d0058d8dbcf6c4f9d7eedab2fc5a3e 2024-11-21T03:41:59,189 INFO [StoreOpener-e6d0058d8dbcf6c4f9d7eedab2fc5a3e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6d0058d8dbcf6c4f9d7eedab2fc5a3e columnFamilyName B 2024-11-21T03:41:59,189 DEBUG [StoreOpener-e6d0058d8dbcf6c4f9d7eedab2fc5a3e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:59,190 INFO [StoreOpener-e6d0058d8dbcf6c4f9d7eedab2fc5a3e-1 {}] regionserver.HStore(327): Store=e6d0058d8dbcf6c4f9d7eedab2fc5a3e/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:41:59,190 INFO [StoreOpener-e6d0058d8dbcf6c4f9d7eedab2fc5a3e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e6d0058d8dbcf6c4f9d7eedab2fc5a3e 2024-11-21T03:41:59,191 INFO [StoreOpener-e6d0058d8dbcf6c4f9d7eedab2fc5a3e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6d0058d8dbcf6c4f9d7eedab2fc5a3e columnFamilyName C 2024-11-21T03:41:59,191 DEBUG [StoreOpener-e6d0058d8dbcf6c4f9d7eedab2fc5a3e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T03:41:59,191 INFO [StoreOpener-e6d0058d8dbcf6c4f9d7eedab2fc5a3e-1 {}] regionserver.HStore(327): Store=e6d0058d8dbcf6c4f9d7eedab2fc5a3e/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T03:41:59,191 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1038): replaying wal for e6d0058d8dbcf6c4f9d7eedab2fc5a3e 2024-11-21T03:41:59,192 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e 2024-11-21T03:41:59,192 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e 2024-11-21T03:41:59,193 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1048): stopping wal replay for e6d0058d8dbcf6c4f9d7eedab2fc5a3e 2024-11-21T03:41:59,193 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1060): Cleaning up temporary data for e6d0058d8dbcf6c4f9d7eedab2fc5a3e 2024-11-21T03:41:59,193 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T03:41:59,194 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1093): writing seq id for e6d0058d8dbcf6c4f9d7eedab2fc5a3e 2024-11-21T03:41:59,196 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T03:41:59,196 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1114): Opened e6d0058d8dbcf6c4f9d7eedab2fc5a3e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63913315, jitterRate=-0.047617390751838684}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T03:41:59,196 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e6d0058d8dbcf6c4f9d7eedab2fc5a3e 2024-11-21T03:41:59,197 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1006): Region open journal for e6d0058d8dbcf6c4f9d7eedab2fc5a3e: Running coprocessor pre-open hook at 1732160519181Writing region info on filesystem at 1732160519181Initializing all the Stores at 1732160519182 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732160519182Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732160519182Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732160519182Cleaning up temporary data from old regions at 1732160519193 (+11 ms)Running coprocessor post-open hooks at 1732160519196 (+3 ms)Region opened successfully at 1732160519197 (+1 ms) 2024-11-21T03:41:59,197 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., pid=124, masterSystemTime=1732160519173 2024-11-21T03:41:59,199 DEBUG [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:41:59,199 INFO [RS_OPEN_REGION-regionserver/fdc0a393186c:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:41:59,199 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, regionState=OPEN, openSeqNum=2, regionLocation=fdc0a393186c,34631,1732160396889 2024-11-21T03:41:59,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=124, ppid=123, state=RUNNABLE, hasLock=false; OpenRegionProcedure e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 because future has completed 2024-11-21T03:41:59,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-21T03:41:59,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; OpenRegionProcedure e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 in 182 msec 2024-11-21T03:41:59,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=123, resume processing ppid=122 2024-11-21T03:41:59,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, ppid=122, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, ASSIGN in 339 msec 2024-11-21T03:41:59,204 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T03:41:59,204 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732160519204"}]},"ts":"1732160519204"} 2024-11-21T03:41:59,205 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T03:41:59,206 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T03:41:59,207 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1970 sec 2024-11-21T03:42:00,108 WARN [regionserver/fdc0a393186c:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-21T03:42:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-21T03:42:00,177 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:42:00,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be1708d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,178 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:42:00,178 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:42:00,179 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:42:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:42:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:42:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1820ccb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:42:00,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:42:00,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:42:00,181 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45044, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:42:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6567c508, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,182 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:42:00,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:42:00,183 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:42:00,184 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:42:00,185 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T03:42:00,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is fdc0a393186c,44285,1732160396080 2024-11-21T03:42:00,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@51ab56a3 2024-11-21T03:42:00,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T03:42:00,187 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T03:42:00,188 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16fe0696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,188 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:42:00,189 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:42:00,189 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:42:00,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:42:00,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:42:00,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@393e2843, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:42:00,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:42:00,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:42:00,190 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:42:00,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58365d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,192 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59de117f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,192 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:42:00,192 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:42:00,192 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:42:00,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:42:00,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:42:00,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71dc4c36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:42:00,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:42:00,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:42:00,194 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45086, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:42:00,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25692321, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,195 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@472a8168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,195 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:42:00,195 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:42:00,195 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:42:00,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:42:00,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:42:00,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3590176f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:42:00,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:42:00,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:42:00,197 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:42:00,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e5e1f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f010827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,198 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:42:00,198 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:42:00,198 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:42:00,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:42:00,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:42:00,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30576c14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:42:00,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:42:00,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:42:00,200 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45108, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:42:00,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@792dc45d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,201 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2822c33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,201 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:42:00,201 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:42:00,202 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:42:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:42:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:42:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35191739, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:42:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:42:00,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:42:00,203 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:42:00,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63aefa0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc5e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,204 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:42:00,204 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:42:00,205 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:42:00,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:42:00,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:42:00,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dbfaa26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:42:00,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:42:00,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:42:00,206 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:42:00,206 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a09f2ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,207 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42daa6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,207 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:42:00,207 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:42:00,207 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:42:00,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:42:00,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:42:00,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b33a7f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,207 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:42:00,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:42:00,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:42:00,208 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:42:00,208 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15981fc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,209 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366fb241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,209 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:42:00,209 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:42:00,209 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:42:00,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:42:00,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:42:00,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1be49ed3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:42:00,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:42:00,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:42:00,210 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:42:00,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@453370ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,211 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e8e518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,211 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:42:00,211 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:42:00,212 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:42:00,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:42:00,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:42:00,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef3b788, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:42:00,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:42:00,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:42:00,213 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:42:00,213 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1be1c400, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,214 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ff7be5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,214 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fdc0a393186c,44285,-1 for getting cluster id 2024-11-21T03:42:00,214 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T03:42:00,214 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1afa9799-9c29-4101-98df-fbf56a76f01a' 2024-11-21T03:42:00,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T03:42:00,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1afa9799-9c29-4101-98df-fbf56a76f01a" 2024-11-21T03:42:00,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78d4c5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fdc0a393186c,44285,-1] 2024-11-21T03:42:00,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T03:42:00,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T03:42:00,215 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T03:42:00,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76058aed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T03:42:00,217 DEBUG [Thread-1963 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:42:00,217 DEBUG [Thread-1961 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:42:00,217 DEBUG [Thread-1978 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:42:00,217 DEBUG [Thread-1970 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:42:00,217 DEBUG [Thread-1976 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:42:00,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:42:00,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:42:00,218 DEBUG [Thread-1967 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:42:00,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:42:00,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:42:00,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:42:00,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:42:00,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:42:00,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:42:00,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:42:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:42:00,219 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:42:00,219 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:42:00,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:42:00,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:42:00,219 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51032, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:42:00,219 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:42:00,219 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51042, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:42:00,220 DEBUG [Thread-1959 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:42:00,220 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51044, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:42:00,220 DEBUG [Thread-1972 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:42:00,220 DEBUG [Thread-1965 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:42:00,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:42:00,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:42:00,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:42:00,221 DEBUG [Thread-1974 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T03:42:00,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:42:00,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:42:00,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:42:00,222 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:42:00,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:42:00,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fdc0a393186c,34631,1732160396889, seqNum=-1] 2024-11-21T03:42:00,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T03:42:00,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:42:00,223 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51066, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:42:00,224 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51052, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:42:00,224 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:42:00,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:42:00,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:42:00,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:42:00,224 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:42:00,225 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T03:42:00,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:42:00,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:42:00,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:42:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-21T03:42:00,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2] 2024-11-21T03:42:00,229 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:42:00,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T03:42:00,230 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:42:00,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:42:00,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on e6d0058d8dbcf6c4f9d7eedab2fc5a3e 2024-11-21T03:42:00,237 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:42:00,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6d0058d8dbcf6c4f9d7eedab2fc5a3e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T03:42:00,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/A/f28166d7c1944f92bbf1b76bb69c554d is 50, key is test_row_0/A:col10/1732160520231/Put/seqid=0 2024-11-21T03:42:00,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51052 deadline: 1732160580270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:00,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51042 deadline: 1732160580274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:00,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:00,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:00,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742261_1437 (size=12001) 2024-11-21T03:42:00,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:00,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51066 deadline: 1732160580274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:00,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1732160580276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:00,285 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:00,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1732160580279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:00,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T03:42:00,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T03:42:00,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:00,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. as already flushing 2024-11-21T03:42:00,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:00,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:00,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:00,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:00,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51052 deadline: 1732160580386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1732160580386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:00,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51042 deadline: 1732160580386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:00,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:00,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:00,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51066 deadline: 1732160580397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:00,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:00,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1732160580397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:00,404 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:00,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:00,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T03:42:00,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:00,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. as already flushing 2024-11-21T03:42:00,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:00,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:00,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:00,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T03:42:00,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:00,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51052 deadline: 1732160580596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:00,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:00,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1732160580596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:00,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:00,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:00,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51042 deadline: 1732160580597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:00,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:00,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:00,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:00,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1732160580607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:00,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:00,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:00,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51066 deadline: 1732160580607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:00,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:00,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/A/f28166d7c1944f92bbf1b76bb69c554d 2024-11-21T03:42:00,688 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T03:42:00,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:00,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. as already flushing 2024-11-21T03:42:00,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:00,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:00,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:00,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/B/504f4aec13f144c989146a60fb9dee7a is 50, key is test_row_0/B:col10/1732160520231/Put/seqid=0 2024-11-21T03:42:00,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742262_1438 (size=12001) 2024-11-21T03:42:00,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T03:42:00,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:00,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. as already flushing 2024-11-21T03:42:00,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:00,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:00,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:00,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:00,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T03:42:00,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:00,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51052 deadline: 1732160580906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:00,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:00,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:00,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1732160580906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:00,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:00,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:00,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51042 deadline: 1732160580906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:00,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:00,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:00,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1732160580917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:00,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:00,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,921 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:00,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51066 deadline: 1732160580917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:00,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:00,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:00,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T03:42:00,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:00,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. as already flushing 2024-11-21T03:42:00,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:00,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:00,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:00,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:01,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/B/504f4aec13f144c989146a60fb9dee7a 2024-11-21T03:42:01,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/C/fc812811ee584bcfb81aa9a8f0b5dca6 is 50, key is test_row_0/C:col10/1732160520231/Put/seqid=0 2024-11-21T03:42:01,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742263_1439 (size=12001) 2024-11-21T03:42:01,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/C/fc812811ee584bcfb81aa9a8f0b5dca6 2024-11-21T03:42:01,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/A/f28166d7c1944f92bbf1b76bb69c554d as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/A/f28166d7c1944f92bbf1b76bb69c554d 2024-11-21T03:42:01,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/A/f28166d7c1944f92bbf1b76bb69c554d, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T03:42:01,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/B/504f4aec13f144c989146a60fb9dee7a as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/B/504f4aec13f144c989146a60fb9dee7a 2024-11-21T03:42:01,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/B/504f4aec13f144c989146a60fb9dee7a, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T03:42:01,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/C/fc812811ee584bcfb81aa9a8f0b5dca6 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/C/fc812811ee584bcfb81aa9a8f0b5dca6 2024-11-21T03:42:01,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T03:42:01,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:01,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. as already flushing 2024-11-21T03:42:01,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:01,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:01,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:01,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:01,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/C/fc812811ee584bcfb81aa9a8f0b5dca6, entries=150, sequenceid=13, filesize=11.7 K 2024-11-21T03:42:01,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e6d0058d8dbcf6c4f9d7eedab2fc5a3e in 915ms, sequenceid=13, compaction requested=false 2024-11-21T03:42:01,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6d0058d8dbcf6c4f9d7eedab2fc5a3e: 2024-11-21T03:42:01,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T03:42:01,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:01,301 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing e6d0058d8dbcf6c4f9d7eedab2fc5a3e 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T03:42:01,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/A/4e6572062c1a4c40b66597fc2a56b744 is 50, key is test_row_0/A:col10/1732160520278/Put/seqid=0 2024-11-21T03:42:01,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742264_1440 (size=12001) 2024-11-21T03:42:01,331 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/A/4e6572062c1a4c40b66597fc2a56b744 2024-11-21T03:42:01,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/B/9857df3e57cc424392884bdb65bacff0 is 50, key is test_row_0/B:col10/1732160520278/Put/seqid=0 2024-11-21T03:42:01,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742265_1441 (size=12001) 2024-11-21T03:42:01,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T03:42:01,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on e6d0058d8dbcf6c4f9d7eedab2fc5a3e 2024-11-21T03:42:01,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. as already flushing 2024-11-21T03:42:01,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1732160581428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:01,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,435 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:01,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51042 deadline: 1732160581428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:01,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51052 deadline: 1732160581429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:01,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:01,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1732160581431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:01,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:01,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:01,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51066 deadline: 1732160581435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:01,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:01,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1732160581537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:01,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:01,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:01,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51042 deadline: 1732160581547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:01,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:01,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51052 deadline: 1732160581547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:01,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:01,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:01,550 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,550 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1732160581547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:01,550 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:01,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:01,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51066 deadline: 1732160581547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:01,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:01,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:01,749 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/B/9857df3e57cc424392884bdb65bacff0 2024-11-21T03:42:01,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:01,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1732160581747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:01,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:01,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/C/103e8105ba474374ae31a1a3f7398f44 is 50, key is test_row_0/C:col10/1732160520278/Put/seqid=0 2024-11-21T03:42:01,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742266_1442 (size=12001) 2024-11-21T03:42:01,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:01,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51042 deadline: 1732160581757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:01,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:01,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:01,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51066 deadline: 1732160581757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:01,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:01,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:01,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51052 deadline: 1732160581757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:01,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:01,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1732160581757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:01,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:02,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:02,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1732160582056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:02,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:02,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51066 deadline: 1732160582067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51052 deadline: 1732160582067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:02,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51042 deadline: 1732160582077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:02,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:02,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1732160582077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:02,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:02,160 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/C/103e8105ba474374ae31a1a3f7398f44 2024-11-21T03:42:02,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/A/4e6572062c1a4c40b66597fc2a56b744 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/A/4e6572062c1a4c40b66597fc2a56b744 2024-11-21T03:42:02,166 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/A/4e6572062c1a4c40b66597fc2a56b744, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T03:42:02,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/B/9857df3e57cc424392884bdb65bacff0 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/B/9857df3e57cc424392884bdb65bacff0 2024-11-21T03:42:02,169 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/B/9857df3e57cc424392884bdb65bacff0, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T03:42:02,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/C/103e8105ba474374ae31a1a3f7398f44 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/C/103e8105ba474374ae31a1a3f7398f44 2024-11-21T03:42:02,172 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/C/103e8105ba474374ae31a1a3f7398f44, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T03:42:02,173 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for e6d0058d8dbcf6c4f9d7eedab2fc5a3e in 872ms, sequenceid=37, compaction requested=false 2024-11-21T03:42:02,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for e6d0058d8dbcf6c4f9d7eedab2fc5a3e: 2024-11-21T03:42:02,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:02,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-21T03:42:02,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-21T03:42:02,175 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-21T03:42:02,175 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9440 sec 2024-11-21T03:42:02,177 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.9520 sec 2024-11-21T03:42:02,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T03:42:02,376 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:42:02,377 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:42:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-21T03:42:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T03:42:02,378 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:42:02,379 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:42:02,379 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:42:02,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T03:42:02,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T03:42:02,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:02,531 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing e6d0058d8dbcf6c4f9d7eedab2fc5a3e 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T03:42:02,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/A/d6b2aceaba28403fb0416d9800d039f3 is 50, key is test_row_0/A:col10/1732160521429/Put/seqid=0 2024-11-21T03:42:02,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742267_1443 (size=12001) 2024-11-21T03:42:02,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on e6d0058d8dbcf6c4f9d7eedab2fc5a3e 2024-11-21T03:42:02,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. as already flushing 2024-11-21T03:42:02,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51066 deadline: 1732160582638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:02,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:02,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51042 deadline: 1732160582639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:02,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:02,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51052 deadline: 1732160582639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:02,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:02,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,649 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:02,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:02,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1732160582640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:02,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:02,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:02,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1732160582641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:02,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:02,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T03:42:02,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:02,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51066 deadline: 1732160582747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:02,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:02,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:02,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51042 deadline: 1732160582757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:02,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:02,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:02,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51052 deadline: 1732160582757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:02,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:02,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1732160582757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:02,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:02,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:02,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:02,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1732160582757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:02,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:02,937 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/A/d6b2aceaba28403fb0416d9800d039f3 2024-11-21T03:42:02,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/B/70c541f0b0fc4f918de72749ae36c39a is 50, key is test_row_0/B:col10/1732160521429/Put/seqid=0 2024-11-21T03:42:02,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742268_1444 (size=12001) 2024-11-21T03:42:02,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:02,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51066 deadline: 1732160582957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:02,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:02,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:02,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51042 deadline: 1732160582967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:02,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:02,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51052 deadline: 1732160582967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:02,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:02,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:02,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:02,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1732160582976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:02,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:02,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1732160582977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:02,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:02,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:02,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:03,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T03:42:03,175 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T03:42:03,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:03,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51066 deadline: 1732160583267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:03,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,274 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:03,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51052 deadline: 1732160583277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:03,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:03,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51042 deadline: 1732160583277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:03,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:03,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:03,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1732160583286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:03,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:03,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1732160583286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:03,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:03,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:03,346 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/B/70c541f0b0fc4f918de72749ae36c39a 2024-11-21T03:42:03,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/C/c7b675a53f434ec0b8d7c1b11b029acd is 50, key is test_row_0/C:col10/1732160521429/Put/seqid=0 2024-11-21T03:42:03,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742269_1445 (size=12001) 2024-11-21T03:42:03,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T03:42:03,755 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/C/c7b675a53f434ec0b8d7c1b11b029acd 2024-11-21T03:42:03,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/A/d6b2aceaba28403fb0416d9800d039f3 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/A/d6b2aceaba28403fb0416d9800d039f3 2024-11-21T03:42:03,761 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/A/d6b2aceaba28403fb0416d9800d039f3, entries=150, sequenceid=49, filesize=11.7 K 2024-11-21T03:42:03,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/B/70c541f0b0fc4f918de72749ae36c39a as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/B/70c541f0b0fc4f918de72749ae36c39a 2024-11-21T03:42:03,764 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/B/70c541f0b0fc4f918de72749ae36c39a, entries=150, sequenceid=49, filesize=11.7 K 2024-11-21T03:42:03,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/C/c7b675a53f434ec0b8d7c1b11b029acd as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/C/c7b675a53f434ec0b8d7c1b11b029acd 2024-11-21T03:42:03,767 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/C/c7b675a53f434ec0b8d7c1b11b029acd, entries=150, sequenceid=49, filesize=11.7 K 2024-11-21T03:42:03,768 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e6d0058d8dbcf6c4f9d7eedab2fc5a3e in 1237ms, sequenceid=49, compaction requested=true 2024-11-21T03:42:03,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for e6d0058d8dbcf6c4f9d7eedab2fc5a3e: 2024-11-21T03:42:03,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:03,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-21T03:42:03,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-21T03:42:03,771 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-21T03:42:03,771 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3900 sec 2024-11-21T03:42:03,772 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.3940 sec 2024-11-21T03:42:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on e6d0058d8dbcf6c4f9d7eedab2fc5a3e 2024-11-21T03:42:03,785 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:42:03,785 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6d0058d8dbcf6c4f9d7eedab2fc5a3e 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T03:42:03,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/A/0ff22d1cf0294c48b0a8209a25bb7398 is 50, key is test_row_0/A:col10/1732160522639/Put/seqid=0 2024-11-21T03:42:03,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742270_1446 (size=14341) 2024-11-21T03:42:03,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:03,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51052 deadline: 1732160583807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:03,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51042 deadline: 1732160583808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:03,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1732160583809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:03,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:03,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1732160583809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:03,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:03,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51066 deadline: 1732160583811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:03,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:03,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:03,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51052 deadline: 1732160583926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:03,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:03,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51042 deadline: 1732160583927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:03,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:03,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:03,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1732160583927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:03,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,934 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:03,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:03,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:03,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51066 deadline: 1732160583927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:03,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:03,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1732160583927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:03,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:03,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:03,935 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:04,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:04,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51052 deadline: 1732160584136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:04,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:04,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51066 deadline: 1732160584137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:04,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:04,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,142 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:04,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:04,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51042 deadline: 1732160584147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:04,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:04,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1732160584147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:04,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1732160584147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,154 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:04,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/A/0ff22d1cf0294c48b0a8209a25bb7398 2024-11-21T03:42:04,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/B/e1a3ee74fa5047e0beb48137ebb0587f is 50, key is test_row_0/B:col10/1732160522639/Put/seqid=0 2024-11-21T03:42:04,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742271_1447 (size=12001) 2024-11-21T03:42:04,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:04,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51066 deadline: 1732160584447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:04,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,454 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:04,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:04,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51052 deadline: 1732160584448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:04,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:04,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:04,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51042 deadline: 1732160584456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:04,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1732160584456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:04,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:04,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1732160584457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:04,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:04,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:04,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T03:42:04,526 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T03:42:04,527 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T03:42:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-21T03:42:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T03:42:04,528 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T03:42:04,529 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T03:42:04,529 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T03:42:04,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/B/e1a3ee74fa5047e0beb48137ebb0587f 2024-11-21T03:42:04,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/C/23188d792c9e41a88b524b610a22024b is 50, key is test_row_0/C:col10/1732160522639/Put/seqid=0 2024-11-21T03:42:04,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742272_1448 (size=12001) 2024-11-21T03:42:04,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T03:42:04,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T03:42:04,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:04,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. as already flushing 2024-11-21T03:42:04,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:04,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:04,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:04,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:04,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T03:42:04,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:04,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. as already flushing 2024-11-21T03:42:04,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:04,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:04,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:04,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:04,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T03:42:04,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:04,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51066 deadline: 1732160584957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:04,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:04,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:04,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51052 deadline: 1732160584967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:04,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:04,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51042 deadline: 1732160584967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:04,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:04,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:04,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1732160584967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:04,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:04,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:04,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:04,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1732160584967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:04,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:04,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:04,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T03:42:04,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:04,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. as already flushing 2024-11-21T03:42:04,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:04,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:04,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:04,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e6d0058d8dbcf6c4f9d7eedab2fc5a3e, NAME => 'TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T03:42:05,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/C/23188d792c9e41a88b524b610a22024b 2024-11-21T03:42:05,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/A/0ff22d1cf0294c48b0a8209a25bb7398 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/A/0ff22d1cf0294c48b0a8209a25bb7398 2024-11-21T03:42:05,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/A/0ff22d1cf0294c48b0a8209a25bb7398, entries=200, sequenceid=74, filesize=14.0 K 2024-11-21T03:42:05,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/B/e1a3ee74fa5047e0beb48137ebb0587f as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/B/e1a3ee74fa5047e0beb48137ebb0587f 2024-11-21T03:42:05,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/B/e1a3ee74fa5047e0beb48137ebb0587f, entries=150, sequenceid=74, filesize=11.7 K 2024-11-21T03:42:05,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/C/23188d792c9e41a88b524b610a22024b as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/C/23188d792c9e41a88b524b610a22024b 2024-11-21T03:42:05,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/C/23188d792c9e41a88b524b610a22024b, entries=150, sequenceid=74, filesize=11.7 K 2024-11-21T03:42:05,027 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for e6d0058d8dbcf6c4f9d7eedab2fc5a3e in 1242ms, sequenceid=74, compaction requested=true 2024-11-21T03:42:05,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e6d0058d8dbcf6c4f9d7eedab2fc5a3e: 2024-11-21T03:42:05,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d0058d8dbcf6c4f9d7eedab2fc5a3e:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T03:42:05,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:42:05,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d0058d8dbcf6c4f9d7eedab2fc5a3e:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T03:42:05,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:42:05,027 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:42:05,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e6d0058d8dbcf6c4f9d7eedab2fc5a3e:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T03:42:05,027 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:42:05,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:42:05,028 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:42:05,028 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:42:05,028 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1541): e6d0058d8dbcf6c4f9d7eedab2fc5a3e/A is initiating minor compaction (all files) 2024-11-21T03:42:05,028 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): e6d0058d8dbcf6c4f9d7eedab2fc5a3e/B is initiating minor compaction (all files) 2024-11-21T03:42:05,028 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d0058d8dbcf6c4f9d7eedab2fc5a3e/A in TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:05,028 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d0058d8dbcf6c4f9d7eedab2fc5a3e/B in TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:05,028 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/B/504f4aec13f144c989146a60fb9dee7a, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/B/9857df3e57cc424392884bdb65bacff0, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/B/70c541f0b0fc4f918de72749ae36c39a, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/B/e1a3ee74fa5047e0beb48137ebb0587f] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp, totalSize=46.9 K 2024-11-21T03:42:05,028 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/A/f28166d7c1944f92bbf1b76bb69c554d, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/A/4e6572062c1a4c40b66597fc2a56b744, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/A/d6b2aceaba28403fb0416d9800d039f3, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/A/0ff22d1cf0294c48b0a8209a25bb7398] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp, totalSize=49.2 K 2024-11-21T03:42:05,028 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting f28166d7c1944f92bbf1b76bb69c554d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732160520226 2024-11-21T03:42:05,028 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 504f4aec13f144c989146a60fb9dee7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732160520226 2024-11-21T03:42:05,029 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e6572062c1a4c40b66597fc2a56b744, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732160520272 2024-11-21T03:42:05,029 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 9857df3e57cc424392884bdb65bacff0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732160520272 2024-11-21T03:42:05,029 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 70c541f0b0fc4f918de72749ae36c39a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732160521419 2024-11-21T03:42:05,029 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6b2aceaba28403fb0416d9800d039f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732160521419 2024-11-21T03:42:05,030 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting e1a3ee74fa5047e0beb48137ebb0587f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732160522639 2024-11-21T03:42:05,030 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ff22d1cf0294c48b0a8209a25bb7398, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732160522638 2024-11-21T03:42:05,036 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d0058d8dbcf6c4f9d7eedab2fc5a3e#B#compaction#379 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:42:05,037 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/B/5d51f83fa1d145dba7cb20eb0bdb7dde is 50, key is test_row_0/B:col10/1732160522639/Put/seqid=0 2024-11-21T03:42:05,037 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d0058d8dbcf6c4f9d7eedab2fc5a3e#A#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:42:05,037 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/A/2d042501d9514b278f7cec7a8acd5416 is 50, key is test_row_0/A:col10/1732160522639/Put/seqid=0 2024-11-21T03:42:05,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742274_1450 (size=12139) 2024-11-21T03:42:05,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742273_1449 (size=12139) 2024-11-21T03:42:05,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=34631 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T03:42:05,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:05,139 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing e6d0058d8dbcf6c4f9d7eedab2fc5a3e 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T03:42:05,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/A/c4961edb7ee04b71bf8cffa8defaead8 is 50, key is test_row_0/A:col10/1732160523808/Put/seqid=0 2024-11-21T03:42:05,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742275_1451 (size=12001) 2024-11-21T03:42:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T03:42:05,450 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/B/5d51f83fa1d145dba7cb20eb0bdb7dde as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/B/5d51f83fa1d145dba7cb20eb0bdb7dde 2024-11-21T03:42:05,454 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6d0058d8dbcf6c4f9d7eedab2fc5a3e/B of e6d0058d8dbcf6c4f9d7eedab2fc5a3e into 5d51f83fa1d145dba7cb20eb0bdb7dde(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:42:05,455 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d0058d8dbcf6c4f9d7eedab2fc5a3e: 2024-11-21T03:42:05,455 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., storeName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e/B, priority=12, startTime=1732160525027; duration=0sec 2024-11-21T03:42:05,455 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T03:42:05,455 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d0058d8dbcf6c4f9d7eedab2fc5a3e:B 2024-11-21T03:42:05,455 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T03:42:05,456 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/A/2d042501d9514b278f7cec7a8acd5416 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/A/2d042501d9514b278f7cec7a8acd5416 2024-11-21T03:42:05,456 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T03:42:05,456 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1541): e6d0058d8dbcf6c4f9d7eedab2fc5a3e/C is initiating minor compaction (all files) 2024-11-21T03:42:05,456 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e6d0058d8dbcf6c4f9d7eedab2fc5a3e/C in TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:05,457 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/C/fc812811ee584bcfb81aa9a8f0b5dca6, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/C/103e8105ba474374ae31a1a3f7398f44, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/C/c7b675a53f434ec0b8d7c1b11b029acd, hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/C/23188d792c9e41a88b524b610a22024b] into tmpdir=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp, totalSize=46.9 K 2024-11-21T03:42:05,457 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting fc812811ee584bcfb81aa9a8f0b5dca6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732160520226 2024-11-21T03:42:05,457 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 103e8105ba474374ae31a1a3f7398f44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732160520272 2024-11-21T03:42:05,457 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting c7b675a53f434ec0b8d7c1b11b029acd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732160521419 2024-11-21T03:42:05,457 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] compactions.Compactor(225): Compacting 23188d792c9e41a88b524b610a22024b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732160522639 2024-11-21T03:42:05,460 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6d0058d8dbcf6c4f9d7eedab2fc5a3e/A of e6d0058d8dbcf6c4f9d7eedab2fc5a3e into 2d042501d9514b278f7cec7a8acd5416(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:42:05,460 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d0058d8dbcf6c4f9d7eedab2fc5a3e: 2024-11-21T03:42:05,460 INFO [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., storeName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e/A, priority=12, startTime=1732160525027; duration=0sec 2024-11-21T03:42:05,460 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:42:05,460 DEBUG [RS:0;fdc0a393186c:34631-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d0058d8dbcf6c4f9d7eedab2fc5a3e:A 2024-11-21T03:42:05,465 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e6d0058d8dbcf6c4f9d7eedab2fc5a3e#C#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T03:42:05,465 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/C/e7746f2e50dc4cddb22ae1fd9dfc1a67 is 50, key is test_row_0/C:col10/1732160522639/Put/seqid=0 2024-11-21T03:42:05,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742276_1452 (size=12139) 2024-11-21T03:42:05,548 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/A/c4961edb7ee04b71bf8cffa8defaead8 2024-11-21T03:42:05,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/B/48108781da00411282ccf08afa69adb7 is 50, key is test_row_0/B:col10/1732160523808/Put/seqid=0 2024-11-21T03:42:05,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742277_1453 (size=12001) 2024-11-21T03:42:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T03:42:05,875 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/C/e7746f2e50dc4cddb22ae1fd9dfc1a67 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/C/e7746f2e50dc4cddb22ae1fd9dfc1a67 2024-11-21T03:42:05,882 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e6d0058d8dbcf6c4f9d7eedab2fc5a3e/C of e6d0058d8dbcf6c4f9d7eedab2fc5a3e into e7746f2e50dc4cddb22ae1fd9dfc1a67(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T03:42:05,882 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e6d0058d8dbcf6c4f9d7eedab2fc5a3e: 2024-11-21T03:42:05,882 INFO [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., storeName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e/C, priority=12, startTime=1732160525027; duration=0sec 2024-11-21T03:42:05,882 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T03:42:05,882 DEBUG [RS:0;fdc0a393186c:34631-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e6d0058d8dbcf6c4f9d7eedab2fc5a3e:C 2024-11-21T03:42:05,957 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/B/48108781da00411282ccf08afa69adb7 2024-11-21T03:42:05,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/C/35f951d403294285a26459a485e7880f is 50, key is test_row_0/C:col10/1732160523808/Put/seqid=0 2024-11-21T03:42:05,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. as already flushing 2024-11-21T03:42:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on e6d0058d8dbcf6c4f9d7eedab2fc5a3e 2024-11-21T03:42:05,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742278_1454 (size=12001) 2024-11-21T03:42:05,982 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/C/35f951d403294285a26459a485e7880f 2024-11-21T03:42:05,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/A/c4961edb7ee04b71bf8cffa8defaead8 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/A/c4961edb7ee04b71bf8cffa8defaead8 2024-11-21T03:42:05,993 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/A/c4961edb7ee04b71bf8cffa8defaead8, entries=150, sequenceid=85, filesize=11.7 K 2024-11-21T03:42:05,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/B/48108781da00411282ccf08afa69adb7 as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/B/48108781da00411282ccf08afa69adb7 2024-11-21T03:42:05,998 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/B/48108781da00411282ccf08afa69adb7, entries=150, sequenceid=85, filesize=11.7 K 2024-11-21T03:42:05,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/C/35f951d403294285a26459a485e7880f as hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/C/35f951d403294285a26459a485e7880f 2024-11-21T03:42:06,003 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/C/35f951d403294285a26459a485e7880f, entries=150, sequenceid=85, filesize=11.7 K 2024-11-21T03:42:06,004 INFO [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=60.38 KB/61830 for e6d0058d8dbcf6c4f9d7eedab2fc5a3e in 864ms, sequenceid=85, compaction requested=false 2024-11-21T03:42:06,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for e6d0058d8dbcf6c4f9d7eedab2fc5a3e: 2024-11-21T03:42:06,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e. 2024-11-21T03:42:06,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fdc0a393186c:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-21T03:42:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44285 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-21T03:42:06,006 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T03:42:06,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e6d0058d8dbcf6c4f9d7eedab2fc5a3e 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T03:42:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(8855): Flush requested on e6d0058d8dbcf6c4f9d7eedab2fc5a3e 2024-11-21T03:42:06,008 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-21T03:42:06,008 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4760 sec 2024-11-21T03:42:06,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.4820 sec 2024-11-21T03:42:06,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45067/user/jenkins/test-data/5a412af3-2499-7878-bff1-3012379655d0/data/default/TestAcidGuarantees/e6d0058d8dbcf6c4f9d7eedab2fc5a3e/.tmp/A/811916e6ae064ea4a63451f82f2cb459 is 50, key is test_row_0/A:col10/1732160526004/Put/seqid=0 2024-11-21T03:42:06,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38147 is added to blk_1073742279_1455 (size=14341) 2024-11-21T03:42:06,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51052 deadline: 1732160586039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:06,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:06,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51042 deadline: 1732160586040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:06,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:06,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:06,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1732160586041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:06,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:06,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:06,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:06,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:06,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1732160586049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:06,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:06,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:06,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:06,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51066 deadline: 1732160586050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:06,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:06,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:06,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:06,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51052 deadline: 1732160586157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:06,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:06,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:06,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:06,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51042 deadline: 1732160586157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:06,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:06,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:06,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:06,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:06,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51024 deadline: 1732160586157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:06,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:06,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:06,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:06,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51002 deadline: 1732160586166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:06,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:06,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:06,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51066 deadline: 1732160586166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:06,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:06,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:06,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:06,176 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 because the exception is null or not the one we care about 2024-11-21T03:42:06,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51052 deadline: 1732160586367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:06,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-21T03:42:06,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2 , the old value is region=TestAcidGuarantees,,1732160518009.e6d0058d8dbcf6c4f9d7eedab2fc5a3e., hostname=fdc0a393186c,34631,1732160396889, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T03:42:06,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34631 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:51042 deadline: 1732160586367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e6d0058d8dbcf6c4f9d7eedab2fc5a3e, server=fdc0a393186c,34631,1732160396889 2024-11-21T03:42:06,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual except